diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..dbb24aa --- /dev/null +++ b/.coveragerc @@ -0,0 +1,28 @@ +[run] +omit = tests/*, aim/storage/migrations/*, aim/storage/env.py + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + + # Don't complain about abstract methods, they aren't run: + @(abc\.)?abstractmethod + + if TYPE_CHECKING: + +[html] +directory = coverage_html_report diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dc9bc5d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.ts linguist-detectable=false +*.tsx linguist-detectable=false +*.scss linguist-detectable=false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d258be --- /dev/null +++ b/.gitignore @@ -0,0 +1,125 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so +*.c + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +pip-wheel-metadata/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +docs/build/ +docs/source/generated + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +aim.sublime-* +.aim/ +.aim-test/ + +.DS_Store +.idea +.vim + +data +examples/.aim +examples/.git + +coverage_html_report/ + +# generated cpp files +# @TODO remove this line once there will be usage of cpp files +aim/storage/**/*.cpp diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..29a9c1b --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,23 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +# reference: https://docs.readthedocs.io/en/stable/config-file/v2.html#sphinx +sphinx: + configuration: docs/source/conf.py + fail_on_warning: false + +# Optionally build your docs in additional formats such as PDF +formats: + - pdf + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.7 + install: + - requirements: requirements.dev.txt + - requirements: docs/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..20e0239 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,1240 @@ +# Changelog + +## 3.19.0 Mar 13, 2024 + +### Enhancements: +- Replace grpc with http/ws as transport for aim tracking server (mihran113) +- Remove `aim storage upgrade 2to3` command (mihran113) +- Allow HF callback to initialize run at on_init_end for tracking custom metrics with callback (dushyantbehl) +- Support artifacts logging and storage in AWS S3 (alberttorosyan) +- Always set run name when initializing Run in lightning callback (martenlienen) + +### Fixes +- Allow the web UI to serve assets symlinked into the static files directory (martenlienen) + +## 3.18.1 Feb 7, 2024 + +### Enhancements: + +- Add support for `sqlalchemy 2.0` (mihran113) +- Add `min/max/first` values tracking and visualization for metrics (mihran113, KaroMourad) + +### Fixes +- Fix pytorch_lightning aliases issue (popfido) +- Fix typos in stat.py to collect gpu memory and power correctly (ChanderG) +- Fix bug in pytorch lightning raising lock timeout (inc0) +- Fix compatibility with `sqlalchemy < 2.0` versions (mihran113) +- Switch to patched version of official `pynvml` (mihran113) +- Remove telemetry tracking (mihran113) + +## 3.17.5 Jun 2, 2023 + +- Fix gpu stat collection when driver is not loaded (mihran113) +- Fix issue with overflowing box content in full-view mode in Base Explorers (KaroMourad) +- Resolve tags list visibility issue in tags page (arsengit) +- Fix issue on git stat collection (mihran113) +- Import `Image` and `Audio` for `TensorboardFolderTracker` (alansaul) +- Extend `aim.ext.tensorboard_tracker.run.Run` to allow stdout logging and system stats and parameter logging (alansaul) +- Add the ability for `TensorboardFolderTracker` to track `Histogram`'s as Aim `Distribution`'s (alansaul) +- Convert NaNs and Infs in responses to strings (n-gao) +- Add activeloop deeplake plugin (drahnreb) + +## 3.17.4 May 4, 2023 + +- Resolve run messages duplication issue for in progress runs (roubkar) +- Fix metric values inconsistency with steps (mihran113) +- Enable CLI for remote repos (mihran113) +- Safe force-acquire index lock using meta-locks (alberttorosyan, mihran113) +- Fix the issue with containers left open (mihran113) +- Fix issue with notebook extension start-up (mihran113) +- Disable SDK events tracking with Segment API (alberttorosyan) + +## 3.17.3 Apr 6, 2023 + +- Fix the community popup overflowing issue (KaroMourad) +- Optimize images blobs URI loading performance (asynclee) + +## 3.17.2 Mar 28, 2023 + +- Fix explorer crashing issue caused by adding a `displayName` property in Grouping component (KaroMourad) + +## 3.17.1 Mar 24, 2023 + +- Avoid explorer crashing when accessing empty chart values (KaroMourad) + +## 3.17.0 Mar 24, 2023 + +### Enhancements +- Expose `run_name` and `run_hash` parameters to the `aim.sdk.adapters.pytorch_lightning.AimLogger` adapter (constd) +- Add navigation link to Experiment page from the Run page (roubkar) +- Add navigation to explorers from the Run page (roubkar) +- Implement Metrics Explorer v2 via Base Explorer (KaroMourad) +- Add Text Explorer to filter and compare text (roubkar) +- Add groundwork for the UI kit v2 for improved usability (arsengit) + +### Fixes + +- Add support for Path type to the repo attribute of the Run class (emekaokoli19) +- Add support for jax>0.4.0 (n-gao) +- Add -y option to Aim CLI commands (emekaokoli19) +- Fix issue with toggling lines visibility during live update (roubkar) +- Fix the issue when HF model doesn't have `num_labels` attribute (mihran113) +- Fix table cell scrolling issue in the Texts tab of the Run page (roubkar) + +## 3.16.2 Mar 3, 2023 + +- Add exception-free mode to Aim (alberttorosyan) +- Expose `capture_terminal_logs` argument for `aim.sdk.adapters` classes (mihran113) +- Handle inconsistency between Sequence data and metadata (alberttorosyan) + +## 3.16.1 Feb 27, 2023 + +- Pin package version `alembic>=1.5.0` (justinvyu) +- Fix segment `flush()` issue with no internet access (alberttorosyan) +- Fix the issue with an empty-illustrations styles on Base explorers (KaroMourad) +- Add 'join community' popup to the sidebar (KaroMourad) +- Use non-strict mode when logging HF model metadata (alberttorosyan) +- Add set() method implementation in ProxyTree/SubtreeView classes (alberttorosyan) + +## 3.16.0 Feb 3, 2023 + +### Enhancements + +- Drop support for python3.6 (mihran113) +- Add support for python3.11 (alberttorosyan) +- Add other x-axis alignment and system logs tracking to cli convert wandb (hjoonjang) +- Add support for pre-binned distribution/histogram (YodaEmbedding) +- Display logged run messages in Run page (VkoHov, alberttorosyan, roubkar) +- Use read-only mode when opening container for indexing (alberttorosyan) +- Add Stable-Baselines3 integration (tmynn) +- Add Acme integration (tmynn) +- Add huggingface/datasets integration (tmynn) +- Enable support for protobuf v4 (mihran113) +- Support events signaling for Remote Tracking server (alberttorosyan) +- Enhance DVC parameters tracking (tmynn) +- Add SDK events tracking (alberttorosyan) +- Add the ability to easily copy run hash with a single click (VkoHov) +- Add Prophet integration (grigoryan-davit) +- Add 'Dataset' type support for hf/datasets (tmynn) +- Add HuggingFace Transformers model info (tmynn) +- Add multidataset logging support for HuggingFace transformers (tmynn) + +### Fixes + +- Fix gpu stats logging when some stats are unavailable (timokau) +- Sub-path support for RTS addresses (mihran113) +- Fix experiment name update issues (mihran113) +- Fix run experiment setting race conditions (mihran113) +- Fix the issue with runs not appearing on UI (mihran113) + +## 3.15.2 Dec 23, 2022 + +- Change logging level for reporter debug messages (alberttorosyan) +- Fix styling issues on the experiment page (KaroMourad) +- Fix client side worker port calculation for RTS (mihran113) +- Add discord community link in the sidebar (arsengit) +- Display experiments descriptions in the explorers tables (arsengit) + +## 3.15.1 Dec 1, 2022 + +- Fix issue with index container lock for older repos (mihran113) +- Fix issue with rendering incorrect empty-illustration content in Audios explorer (KaroMourad) + +## 3.15.0 Nov 26, 2022 + +### Enhancements: + +- Implement Aim callbacks system and extended notifications (alberttorosyan) +- Add chart legends to the Metrics Explorer (KaroMourad) +- Implement vertically scalable version of Remote Tracking (mihran113, alberttorosyan) +- Add the ability to search, filter, and compare audio through Audios Explorer (VkoHov) +- Add epoch tracking for PyTorch Lightning (tmynn) +- Add PaddlePaddle integration (tmynn) +- Add Optuna integration (tmynn) +- Use `packaging` to parse version strings (jangop) +- Implement the experiment page for the overall experiment info view (VkoHov) +- Implement dynamic flushing mechanism for `CheckIn`s based on the flag (mahnerak) +- Implement robust locking and indexing mechanism for Aim Runs (alberttorosyan) + +### Fixes: + +- Fix multiple progress bars handling for terminal logs capturing (mihran113) +- Handle resources when multiple `Ctrl-C`s are pressed (alberttorosyan) +- Remove non unicode symbols from `aim up` command logs (mihran113) +- Fix "Show Table Diff" for list type elements in runs, params and scatters explorers (kumarshreshtha) +- Support non-Latin chars for encoding in Aim UI (roubkar) +- Make new `CheckIn`s always override the expiry date, consistent to what is documented (mahnerak) + +## 3.14.4 Nov 11, 2022 + +- Fix dropdowns' selected options losses in time of searching other options in Figures Explorer (rubenaprikyan) +- Fix the group property name visibility in the images and audio tabs (VkoHov) +- Change the color contrast of the icons in the manage columns popover (VkoHov) +- Add notifier config files to aim package (alberttorosyan) +- Fix audios to numpy conversion (mihran113) + +## 3.14.3 Oct 29, 2022 + +- Fix search for empty queries in explorers (KaroMourad) + +## 3.14.2 Oct 28, 2022 + +- Add support to sync explorer state through url on Base and Figures Explorers (rubenaprikyan) +- Add support to highlight syntax error in Figures Explorer (KaroMourad) +- Fix issue with applying solid stroke styles on stroke badge in table (KaroMourad) +- Fix active runs indicators overlapping issue in LineChart (KaroMourad) +- Add support for text style formatting in the logs tab (VkoHov) +- Fix "`TypeError: check()` keywords must be strings" for `Run.metrics()` method (alberttorosyan) +- Fix run info API call error when tag color/description is None (alberttorosyan) +- Fix remote heartbeat resource cleanup (mihran113) + +## 3.14.1 Oct 7, 2022 + +- Fix the current release duplication highlighting issue on the Dashboard page (arsengit) + +## 3.14.0 Oct 6, 2022 + +### Enhancements: + +- Move `aim reindex` command under `aim storage` group (mihran113) +- Add the ability to attach/remove tags on the Run Page (roubkar) +- Support dictionary as an argument of `Run.track` (alberttorosyan) +- Display the tags of the run in the tables of the explorers (VkoHov) +- Revamp Figures explorer controls and grouping sections for better onboarding and usability (VkoHov, KaroMourad) +- Replace the spinner loader with a lighter one (VkoHov) +- Add fast.ai integration (tmynn) +- Add command for dangling params cleanup (mihran113) +- Add top and bottom appearance modes to the chart popover (VkoHov) +- Deprecate Python 3.6 (alberttorosyan) +- Add MXNet integration (tmynn) +- Create a Dashboard page to provide a better onboarding experience (arsengit, roubkar, KaroMourad, mihran113) +- Add support for tracking jax device arrays (mihran113) + +### Fixes: + +- Fix chart hovering issue occurring when "nan" values are tracked (KaroMourad) +- Use empty dict as default when getting Run params (alberttorosyan) +- Change unit-tests data isolation mechanism (alberttorosyan) +- Adjust the visibility of the run color in tables (VkoHov) +- Fix response headers for remote tracking server (mihran113) +- Fix `TypeError`s in single run page (mihran113) + +## 3.13.4 Sep 25, 2022 + +- Add the ability to disable smoothing explicitly (KaroMourad) +- Virtualize the run params list in the Run page (roubkar) + +## 3.13.3 Sep 16, 2022 + +- Fix request cancellation on `Logs` tab (mihran113) +- Fix the data live update handling in the Logs tab (VkoHov) + +## 3.13.2 Sep 10, 2022 + +- Fix content overlapping issue of x-axis alignment dropdown (KaroMourad) +- Fix the regression line rendering issue on Scatter plot exported image (KaroMourad) + +## 3.13.1 Sep 1, 2022 + +- Add support for querying metrics by last value (mihran113) +- Fix aim reindex command failure (alberttorosyan) +- Fix issue with remote runs re-open (mihran113) +- Deprecate custom set Run.hash values (alberttorosyan) +- Tune mlflow converter run properties (tmynn) +- Fix `AimLogger` deprecation issues related to release of PyTorch Lightning v1.7 (djwessel) + +## 3.13.0 Aug 21, 2022 + +### Enhancements: + +- Add Figures Explorer to visualize and compare plotly figures (rubenaprikyan, KaroMourad, arsengit, VkoHov, roubkar) +- Add Base Explorer as core of all explorers (rubenaprikyan, KaroMourad, arsengit, VkoHov, roubkar) +- Add logging for remote resource cleanup and network stability (mihran113) +- Restrict Run.hash to auto-generated values only (alberttorosyan) +- Add ability to compare selected runs from the table (arsengit) +- Notify users about failed/stalled runs (mahnerak, alberttorosyan) +- Add ability to pin metrics in Run Page (mihran113, roubkar) +- Add step for unit tests for nightly releases workflow (mihran113) +- Add Keras-Tuner integration (tmynn) +- Add Weights & Biases to Aim log converter (tmynn) + +### Fixes: + +- Fix chart exporting issue (KaroMourad) +- Fix aim ui rendering issue on notebooks (rubenaprikyan) +- Fix live update retry to show live data after solving connection problems with the server (rubenaprikyan) +- Fix tensorboard convert while converting tensor (sharathmk99) +- Fix incorrect column keys of metrics in the table grid of the runs dashboard (VkoHov) +- Fix git info collection (mihran113) +- Fix code block content and query copying functionality (arsengit) +- Provide compatibility between plotly and matplotlib (tmynn) +- Warn to use aim.Image if aim.Figure fails (tmynn) + +## 3.12.2 Aug 5, 2022 + +- Fix formatting of empty metric contexts (VkoHov) +- Apply lazy loading on metrics in Run Page (roubkar) + +## 3.12.1 Aug 2, 2022 + +- Loosen version requirements for grpcio (alberttorosyan) +- Fix remote heartbeat-watcher resource cleanup (mihran113) +- Break long metric names into multiple lines in Run Page (roubkar) +- Enable run filtering by metric values (mihran113) +- Fix Cython version to eliminate build errors (mihran113) + +## 3.12.0 Jul 22, 2022 + +### Enhancements: + +- Add ability to set axes range manually for line charts on UI (KaroMourad) +- Add more user-friendly querying for dates (mihran113, arsengit) +- Filter redundant tooltip data from URL config state (KaroMourad) +- Improve rendering performance by enhancing table columns virtualization mechanism (roubkar) +- Increase visibility and usability of the Show table diff button (arsengit) +- Add support for tensorboard audios conversion (mihran113) +- Format params keys/paths properly (VkoHov) +- Mention explicitly run params everywhere params is mentioned (VkoHov) +- Add ability to hide a batch of items in explorers (VkoHov) +- Add ability to sort by the last value of the metric in table (VkoHov) +- Preserve active line even if it is dropped out of the filtered area (VkoHov) +- Add run duration property for SDK and queries (mihran113) +- Add client vs server version check for remote tracking server (mihran113) +- Add Remote tracking client heartbeat (mihran113) + +### Fixes: + +- Tune table sorting icon box overlapping with column box in compact mode (KaroMourad) +- Fix tensorboard log conversion for images (mihran113) +- Check if gradient is None when tracking gradient distributions (kage08) +- Fix displaying non-syntax errors across Aim UI (arsengit) +- Fix queries on remote repos (mihran113) +- Fix interval progress reports for query apis (mihran113) +- Fix query request cancellation errors (mihran113) +- Auto-detect and address inconsistencies in meta and series trees (mahnerak) + +## 3.11.2 Jul 8, 2022 + +### Enhancements: + +- Display the error position when getting syntax errors after searching (arsengit) + +### Fixes: + +- Avoid saving crashed or terminated search requests as the last state on explorers (arsengit) +- Remove the progress bar blinking when searching runs in Runs Explorer (KaroMourad) +- Fix the "matched runs" sentence color style in progress bars (KaroMourad) +- Fix `SyntaxError` handling for python3.10+ (mihran113) +- Fix generic Exceptions handling and adjust HTTPException handling (alberttorosyan) + +## 3.11.1 Jun 27, 2022 + +- Replace base58 encoder with base64 (KaroMourad, VkoHov) +- Fix Notes tab loading issue (arsengit) +- Fix the loading logic of the `monaco editor` across the Aim Ui (arsengit) +- Fix `Table` export functionality in Params and Scatters explorers (arsengit) +- Allow mixing numeric types on a single Sequence (alberttorosyan) + +## 3.11.0 Jun 21, 2022 + +### Enhancements: + +- Add `--uds` option for `aim up` command (mihran113) +- Add progress reporting for search APIs and tqdm progress for SDK queries (mihran113) +- Add all the attributes of runs in the grouping popovers (KaroMourad) +- Display progress bar on Explorer pages when searching metadata (KaroMourad) +- Improve the processing speed for tb to aim converter (osoblanco) +- Adjust charts hover attributes position calculation and styles (KaroMourad) +- Improve formatting of numbers by setting maximum precision (KaroMourad) +- Add cloud storage backups to AWS S3 for aim repo runs (karan2801) +- Add LightGBM integration example (gorarakelyan) +- Add descriptive document titles for pages (KaroMourad) +- Implement unit-tests for aim SDK utils (yeghiakoronian) +- Display std.dev/err aggregated values in the table (VkoHov) +- Add `active` state indicator property for `aim.Run` (mihran113) +- Add `active` state indicators on the chart (VkoHov) +- Add ability to edit run name and description of run (VkoHov) +- Show the description in the sidebar of the run overview tab (VkoHov) +- Add all the attributes of run in the tooltip (VkoHov) +- Optimize the initial render time of Aim UI by using more lightweight font-family (arsengit) +- Use monaco editor as the syntax highlighter across the Aim UI (arsengit) +- Add loader to the top of the logs box in the run page (VkoHov) +- Add the date and the duration of run in the header of the single run page (VkoHov) +- Add the name, status and duration of run in the runs table of the tags page (VkoHov) +- Fit long name values in manage columns popover (arsengit) +- Add caching mechanism for sequence queries to optimize query performance (mihran113) +- Use step random hash as a key for metric sequences (alberttorosyan) + +### Fixes: + +- Fix issue with tensorboard to aim conversion (osoblanco) +- Fix reset zoom history on alignment type change (KaroMourad) +- Fix issue with rendering incorrect data when x-axis aligned by `relative time/epoch` (KaroMourad) +- Fix LineCart axis ticks overlapping issue on log scale (KaroMourad) +- Change zooming default option to multiple (VkoHov) +- Change grouped rows' min and max values names to `Group Min` and `Group Max` (VkoHov) +- Preserve the search input value of the grouping dropdown (VkoHov) +- Change the titles and placeholders in popovers (VkoHov) +- Resolve typing latency issue in the query search input (arsengit) +- Reorder and add non-hideable table columns (arsengit) +- Change the font of the runs navigation popover (VkoHov) +- Keep color persistence state after page reload (VkoHov) +- Resolve content blinking issue after search in the run page (arsengit) +- Fix scroll to bottom on live-update in logs tab (VkoHov) +- Fix timezone issues for activity map (mihran113) +- Fix `aim up` command output when `--port 0` is passed (mihran113) + +## 3.10.3 May 31, 2022 + +- Adjust the content overflowing of the Delete and the Archive modals (VkoHov) +- Resolve issue with redirect in run page (arsengit) + +## 3.10.2 May 26, 2022 + +- Adjust SRP Logs row height calculation (VkoHov) +- Fix issue with live update requests scheduler (rubenaprikyan) +- Fix log capturing crash during run garbage collection (mihran113) +- Fix Pytorch Lightning adapter `finalize` method (mihran113) +- Fix params duplication in dropdowns (VkoHov) +- Skip system params in Explorer pages (alberttorosyan) + +## 3.10.1 May 18, 2022 + +- Resolve issue with rendering run params in the overview tab of SRP (arsengit) +- Fix issue with search query state update (arsengit) + +## 3.10.0 May 17, 2022 + +### Enhancements: + +- Add ability to adjust the density of the visible content in tables (roubkar) +- Set `metric.name` as default option for grouping (roubkar) +- Show user-selected params before group config in chart popover (roubkar) +- Optimize stream decoding performance on UI (mahnerak) +- Add support for animated image formats to Aim Image object (devfox-se) +- Add `AimLogger` for Catboost (devfox-se) +- Add `AimCallback` for LightGBM (devfox-se) +- Keep the extents of `HighPlot` axes brush in the state and the URL (VkoHov) +- Integrate `aim` with `cimport`-able `aimrocks` (mahnerak) +- Add `__slots__` to some classes to improve performance (mahnerak) +- Define base abstractions for `Iterator` and `DB` by borrowing from `aimrocks` (mahnerak) +- Use `KeysIterator` and `ValuesIterator` wrappers instead of reimplementing (mahnerak) +- Rename `PrefixView.container` to `PrefixView.parent` (mahnerak) +- Reimplement `absolute_path` (mahnerak) +- Cython bindings for `PrefixView`, `TreeView`, `Container`, `ArrayView` (mahnerak) +- Add ability to track and visualize stdout/stderr (mihran113, VkoHov) +- Fix `AimLogger` deprecation issues related to release of PyTorch Lightning v1.5 (arnauddhaene) +- Enable better autocomplete experience with monaco editor (arsengit) +- Pre-loading and caching necessary resources, add pre-loader animation to Aim UI (arsengit) + +### Fixes: + +- Remove hard-coded installation of pre-requirements (mahnerak) +- Remove duplicate code from `TreeView` and `Container` methods (mahnerak) +- Fix issue with filtering metrics values in single run page (KaroMourad) + +## 3.9.4 May 12, 2022 + +- Fix run remote tracking queue cleanup (mihran113) +- Fix HF callback before training access (mihran113) +- Fix compatibility with Jinja 3.1 (devfox-se) + +## 3.9.3 May 10, 2022 + +- Fix affecting stroke types after changing color persistence (KaroMourad) + +## 3.9.2 Apr 29, 2022 + +- Move aim_ui package data to separate directory (devfox-se) + +## 3.9.1 Apr 29, 2022 + +- Move aim_ui package data to separate directory (devfox-se) + +## 3.9.0 Apr 29, 2022 + +### Enhancements: + +- Add `Notes Tab` to single run page (arsengit) +- Add the run name to the batch delete and the batch archive modals (VkoHov) +- Increase the scalability of rendering lines in charts (KaroMourad) +- Increase live update requests delay to prevent performance issues (rubenaprikyan) +- Change font-family to monospace in the Table component (arsengit) +- Add info massage for single value sliders (VkoHov) +- Add `--log-level` argument for aim up/server commands (mihran113) +- Add notes backend api interface (devfox-se) +- Fix type hints in `Repo` class (uduse) + +### Fixes: + +- Fix LineChart y-dimension margin calculation (KaroMourad) +- Fix HighPlot lines partially rendering issue (KaroMourad) +- Fix HighPlot axis ticks overlapping issue (KaroMourad) +- Fix sorting Params/Scatters explorer axis ticks (KaroMourad) +- Fix compatibility with pytorch-lightning v1.6.0 (mihran113) +- Fix the image's original size cropping (VkoHov) +- Fix `PATH` related issues for `alembic` and `uvicorn` (mihran113) +- Fix queries for custom object APIs (mihran113) +- Fix chart height updating when resize mode changed (VkoHov) +- Fix HuggingFace callback context capturing (mihran113) +- Fix Params/Scatters explorers' row hiding functionality (VkoHov) +- Fix Profiler logs are saved outside repo directory (devfox-se) + +## 3.8.1 Apr 6, 2022 + +- Encode run hash before including in CSS selectors (Hamik25) +- Fix displaying incorrect metric values for large range scale in LineChart (KaroMourad) +- Fix issue with rendering lines for large range scale in LineChart (KaroMourad) +- Fix issue with URL state sync for bookmarks (roubkar) +- Fix issue with displaying negative param values on Aim UI (roubkar) +- Fix row hiding functionality (roubkar) +- Tune RunOverviewTab container styles (arsengit) +- Update documentations links on UI (rubenaprikyan) +- Fix `RepoIndexManager` run's reference cleanup (mihran113) +- Fix remote run finalization (mihran113) +- Fix issue with fetch on load more (infinite scroll) functionality in Runs Explorer (rubenaprikyan) + +## 3.8.0 Mar 26, 2022 + +### Enhancements: + +- Hugging Face adapter refactoring (mihran113) +- Add run description columns to all run specific tables (VkoHov, mihran113) +- Change images rendering optimization default value to smoother (VkoHov) +- Set default steps ordering to desc in single run tabs (VkoHov, devfox-se) +- Add run name to grouping, ordering and run navigation popovers (VkoHov) +- Add ability to apply color scale on columns with numeric values (VkoHov) +- Refactored XGBoost AimCallback (devfox-se) +- Reopenable callbacks for integrations (mihran113) +- Add DVC integration (devfox-se) +- Add API profiler and unified API error response (devfox-se) +- Add API to retrieve N'th step of sequence (devfox-se) + +### Fixes: + +- Fix issue with calculation of active point on mouse hover in the LineChart (KaroMourad) +- Fix issue with wrong URL caching for Explorer pages (roubkar) +- Fix issue with focusing on the chart active point while moving the cursor (KaroMourad) +- Fix the image full view toggle icon visibility if the image has a white background (VkoHov) +- Fix scroll to the end of the audio tab (VkoHov) +- Add scrollbar to image full view mode content (VkoHov) +- Fix issues with run name/description not being set (mihran113) +- Fix issue with run single page tabs result caching (mihran113) +- Fix git system param tracking (devfox-se) +- Fix runs manual closing (mihran113) +- Fix Docker image creation step in packaging workflow (alberttorosyan) +- Fix Jinja2 template rendering with starlette==0.14.2 (alberttorosyan) + +## 3.7.5 Mar 18, 2022 + +- Add request aborting functionality in single run page tabs (arsengit) +- Render plotly figures properly in single run page (arsengit) + +## 3.7.4 Mar 15, 2022 + +- Fix density min and max validation calculation (VkoHov) + +## 3.7.3 Mar 14, 2022 + +- Add missing names for dynamically imported files in single run page (arsengit) + +## 3.7.2 Mar 10, 2022 + +- Fix issue with rendering UI re keeping long URL (KaroMourad) +- Split code in the single run page to optimize chunk size (arsengit) + +## 3.7.1 Mar 10, 2022 + +- Fix metric queries with epoch=None (alberttorosyan) + +## 3.7.0 Mar 9, 2022 + +### Enhancements: + +- Add Run overview tab in run single page (arsengit, VkoHov, KaroMourad, rubenaprikyan) +- Custom max message size for Aim Remote tracking (alberttorosyan) +- Docker images for aim up/server (alberttorosyan) +- TF/Keras adapters refactoring (mihran113) +- Remote tracking client-side retry logic (aramaim) +- Add record_density to initial get-batch request for figures (VkoHov) + +### Fixes: + +- Fix rendering new lines in texts visualizer (arsengit) + +## 3.6.3 Mar 4, 2022 + +- Fix UI rendering issue on colab (rubenaprikyan) + +## 3.6.2 Mar 2, 2022 + +- Fix chart interactions issue in the Single Run Page Metrics tab (roubkar) +- Fix `resolve_objects` in remote tracking client subtree (alberttorosyan) +- Reject `0` as step/record count (alberttorosyan, VkoHov) +- Fix error on mlflow conversion by experiment id (devfox-se) + +## 3.6.1 Feb 25, 2022 + +- Fix issue with aligning x-axis by custom metric (KaroMourad) +- Add `__AIM_PROXY_URL__` env variable to see full proxy url when running `aim up` command(rubenaprikyan) +- Add `--proxy-url` argument to notebook extension's `%aim up` to render UI correctly if there is a proxy server (rubenaprikyan) +- Add SageMaker integration, `jupyter-server-proxy` s bug-fix script (rubenaprikyan, mahnerak) +- Fix animation support in Plotly visualization and figure loading performance (Hamik25, mihran113) +- Display `None` values in group config column (VkoHov, Hamik25) +- Fix rendering issue on `Select` form search suggestions list (arsengit) +- Fix PL.AimLogger save_dir AttributeError (GeeeekExplorer) +- Remove `__example_type__` substring from param name (VkoHov) + +## 3.6.0 Feb 22 2022 + +### Enhancements: + +- Sort params columns in alphabetical order (arsengit) +- Add illustrations for indicating explorer search states (arsengit) +- Ability to export chart as image (KaroMourad) +- Ability to group by metric.context (VkoHov) +- Tune manage columns items highlighting styles (VkoHov) +- Set active style on table actions popover buttons with applied changes (arsengit) +- Unification of Run Custom Object APIs (alberttorosyan, VkoHov) +- Aim repo runs data automatic indexing (alberttorosyan) +- Pytorch Lightning adapter refactoring (mihran113) +- Add Pytorch Ignite integration (mihran113) +- Add wildcard support for `aim runs` subcommands (mihran113) +- Add MLflow logs conversion command (devfox-se) +- Add CustomObject implementation for `hub.dataset` (alberttorosyan) + +### Fixes: + +- Fix live updated data loss after triggering endless scroll (VkoHov) +- Fix system metric columns pinning functionality and grouping column order (arsengit) +- Fix system metrics search in manage columns popover (VkoHov) +- Fix queries on remote repos (mihran113) +- Fix incorrect boolean value formatting (VkoHov) + +## 3.5.4 Feb 15 2022 + +- Fix batch archive functionality (VkoHov) +- Add repo lock/release feature (devfox-se) + +## 3.5.3 Feb 11 2022 + +- Fix rendering issue in runs explorer page (arsengit) + +## 3.5.2 Feb 10 2022 + +- Fix issue with displaying current day activity cell on week's first day (rubenaprikyan) +- Fix issue with filtering options while typing in input of autocomplete in Tooltip and Grouping popovers (rubenaprikyan) + +## 3.5.1 Feb 4 2022 + +- Fix folder creation when tracking with remote tracker (aramaim) + +## 3.5.0 Feb 3 2022 + +### Enhancements: + +- Ability to hide system metrics from table (arsengit) +- Add input validations to range selectors (Hamik25) +- Improve media panel rendering performance on hovering over images (KaroMourad) +- Add ability to parse and import TensorFlow events into aim (devfox-se) +- Add system parameter logging: CLI, Env, Executable, Git, Installed packages (devfox-se) +- Convert nested non-native objects (e.g. OmegaConf config instance) upon storing (devfox-se) +- Add cli subcommands cp and mv for aim runs command (mihran113) +- Add handler for matplotlib figures in Image and Figure custom objects (devfox-se) +- Improve highlighting of table focused/hovered/selected row (VkoHov) + +### Fixes: + +- Fix stalled runs deletion (mihran113) +- Fix background transparency in colab when using dark mode of system (rubenaprikyan) +- Fix Grouping and Tooltip popovers states' resetting issue when live-update is on (rubenaprikyan) +- Fix table column's sort functionality issue in Params and Scatters Explorers (rubenaprikyan) + +## 3.4.1 Jan 23 2022 + +- Fix issue with displaying experiment name in Images Explorer table (VkoHov) + +## 3.4.0 Jan 22 2022 + +- Add ability to apply group stacking on media elements list (KaroMourad) +- Add ability to apply sorting by run creation_time on table rows (roubkar) +- Add ability to filter texts table with keyword matching (roubkar, rubenaprikyan) +- Add ability to delete run from settings tab (Hamik25) +- Enhance controls states of explorer pages (arsengit) +- Add --repo, --host arguments support for notebook extension (VkoHov, rubenaprikyan) +- Add trendline options to ScatterPlot (roubkar) +- Add ability to display images in original size and align by width (arsengit) +- Add version, docs and slack links to sidebar (arsengit) +- Enhance AudioPlayer component (arsengit) +- Recover active tab in run details page after reload (roubkar) +- Add ability to archive or delete runs with batches (VkoHov) +- Remote tracking server [experimental] (alberttorosyan, mihran113, aramaim) +- Add ability to change media elements order (VkoHov) +- Add ability to hard delete runs (alberttorosyan) +- Lossy format support for aim.Image (devfox-se) +- Timezone issues fix for creation and end times (mihran113) + +## 3.3.5 Jan 14 2022 + +- Add non-strict write mode to replace not-yet-supported types with their + string representations. (mahnerak) +- Log pytorch_lightning hyperparameters in non-strict mode. (mahnerak) + +## 3.3.4 Jan 10 2022 + +- Fix issue with WAL files flushing (alberttorosyan) +- Support for omegaconf configs in pytorch_lightning adapter (devfox-se) + +## 3.3.3 Dec 24 2021 + +- Fix issue with showing range panel in Images Explorer (roubkar) + +## 3.3.2 Dec 20 2021 + +- Fix issue with not providing point density value to live-update query (rubenaprikyan) + +## 3.3.1 Dec 18 2021 + +- Fix getValue function to show correct chart title data (KaroMourad) + +## 3.3.0 Dec 17 2021 + +- Add ability to track and explore audios in run detail page (arsengit, VkoHov, devfox-se) +- Add ability to track and visualize texts (mihran113, roubkar) +- Fix boolean values encoding (mahnerak) +- Add Scatter Explorer to visualize correlations between metric last value and hyperparameter (KaroMourad) +- Add ability to track and visualize plotly objects (devfox-se, Hamik25, rubenaprikyan) +- Add ability to query distributions by step range and density (VkoHov, rubenaprikyan) +- Add colab notebook support (mihran113, rubenaprikyan) +- Implement images visualization tab in run detail page (VkoHov, KaroMourad) +- Add custom URL prefix support (mihran113, Hamik25, roubkar) +- Enhance metric selection dropdowns to see lists in alphabetical order (rubenaprikyan) + +## 3.2.2 Dec 10 2021 + +- Fix Run finalization index timeout issue (alberttorosyan) + +## 3.2.1 Dec 8 2021 + +- Add ability to provide custom base path for API (mihran113, roubkar) +- Fix table groups column default order (arsengit) +- Fix table panel height issue in runs explorer page (arsengit) + +## 3.2.0 Dec 3 2021 + +- Add ability to cancel pending request (roubkar, arsengit) +- Add support for secure protocol for API calls (mihran113, roubkar) +- Implement image full size view (VkoHov) +- Add ability to manipulate with image size and rendering type (arsengit) +- Enhance Table column for selected grouping config options (arsengit) +- Implement suggestions list for AimQL search (arsengit, rubenaprikyan) +- Add ability to track and visualize distributions (mihran113, rubenaprikyan) +- Add notebook extension, magic functions (rubenaprikyan) + +## 3.1.1 Nov 25 2021 + +- Apply default ordering on images set (VkoHov) +- Ability to show image data in a tooltip on hover (KaroMourad) +- Support of Image input additional data sources (alberttorosyan) +- Ability to export run props as pandas dataframe (gorarakelyan) +- Slice image sequence by index for the given steps range (alberttorosyan) +- Improve Images Explorer rendering performance through better images list virtualization (roubkar) + +## 3.1.0 Nov 20 2021 + +- Add ability to explore tracked images (VkoHov) +- Improve rendering performance by virtualizing table columns (roubkar) +- Add ability to apply grouping by higher level param key (roubkar) +- Add ability to specify repository path during `aim init` via `--repo` argument (rubenaprikyan) + +## 3.0.7 Nov 17 2021 + +- Fix for missing metrics when numpy.float64 values tracked (alberttorosyan) + +## 3.0.6 Nov 9 2021 + +- Fix for blocking container optimization for in progress runs (alberttorosyan) + +## 3.0.5 Nov 9 2021 + +- Add tqdm package in setup.py required section (mihran113) + +## 3.0.4 Nov 8 2021 + +- Switch to aimrocks 0.0.10 - exposes data flushing interface (mihran113) +- Optimize stored data when runs finalized (mihran113) +- Update `aim reindex` command to run storage optimizations (alberttorosyan) +- Storage partial optimizations on metric/run queries (alberttorosyan) + +## 3.0.3 Nov 4 2021 + +- Bump sqlalchemy version to 1.4.1 (alberttorosyan) + +## 3.0.2 Oct 27 2021 + +- Switch to aimrocks 0.0.9 - built on rocksdb 6.25.3 (alberttorosyan) +- Remove grouping select options from Params app config (VkoHov) +- Sort metrics data in ascending order for X-axis (KaroMourad) + +## 3.0.1 Oct 22 2021 + +- Check telemetry_enabled option on segment initialization (VkoHov) +- Draw LineChart Y-axis (horizontal) tick lines on zooming (KaroMourad) +- Sort select options/params based on input value (roubkar) +- Fix query construction issue for multiple context items (roubkar) +- Fix issue with making API call from Web Worker (VkoHov) + +## 3.0.0 Oct 21 2021 + +- Completely revamped UI: + + - Runs, metrics and params explorers + - Bookmarks, Tags, Homepage + - New UI works smooth with ~500 metrics displayed at the same time with full Aim table interactions + +- Completely revamped storage: + - 10x faster embedded storage based on Rocksdb + - Average run query execution time on ~2000 runs: 0.784s + - Average metrics query execution time on ~2000 runs with 6000 metrics: 1.552s + +## 2.7.1 Jun 30 2021 + +- Fix bookmark navigation issue (roubkar) +- Empty metric select on X-axis alignment property change (roubkar) + +## 2.7.0 Jun 23 2021 + +- Add ability to export table data as CSV (KaroMourad) +- Add ability to bookmark explore screen state (roubkar) +- Add dashboards and apps API (mihran113) + +## 2.6.0 Jun 12 2021 + +- Resolve namedtuple python 3.5 incompatibility (gorarakelyan) +- Add ability to align X-axis by a metric (mihran113, roubkar) +- Add tooltip popover for the chart hover state (roubkar) + +## 2.5.0 May 27 2021 + +- Set gunicorn timeouts (mihran113) +- Remove redundant deserialize method (gorarakelyan) +- Move the Flask server to main repo to support 'docker'less UI (mihran113) + +## 2.4.0 May 13 2021 + +- Bump up Aim UI to v1.6.0 (gorarakelyan) +- Add xgboost integration (khazhak) +- Update keras adapter interface (khazhak) +- Convert tensors to python numbers (gorarakelyan) + +## 2.3.0 Apr 10 2021 + +- Bump up Aim UI to v1.5.0 (gorarakelyan) +- Set default interval of sys tracking to 10 seconds (gorarakelyan) +- Add ability to track system metrics (gorarakelyan) + +## 2.2.1 Mar 31 2021 + +- Bump up Aim UI to v1.4.1 (gorarakelyan) + +## 2.2.0 Mar 24 2021 + +- Bump up Aim UI to v1.4.0 (gorarakelyan) +- Add Hugging Face integration (Khazhak) +- Reorganize documentation (Tatevv) + +## 2.1.6 Feb 26 2021 + +- Add ability to opt out telemetry (gorarakelyan) +- Remove experiment name from config file when calling repo.remove_branch method (gorarakelyan) + +## 2.1.5 Jan 7 2021 + +- Handle NaN or infinite floats passed to artifacts (gorarakelyan) + +## 2.1.4 Dec 2 2020 + +- Add ability to specify session run hash (gorarakelyan) +- Initialize repo if it was empty when opening session (gorarakelyan) +- Add validation of map artifact parameters (gorarakelyan) + +## 2.1.3 Nov 24 2020 + +- Support comparison of list type contexts (gorarakelyan) + +## 2.1.2 Nov 24 2020 + +- Fix empty contexts comparison issue (gorarakelyan) + +## 2.1.1 Nov 22 2020 + +- Return only selected params in SelectResult (gorarakelyan) + +## 2.1.0 Nov 19 2020 + +- Add AimRepo select method (gorarakelyan) +- Implement SelectResult class (gorarakelyan) + +## 2.0.27 Nov 13 2020 + +- Fix issue with artifact step initializer (gorarakelyan) + +## 2.0.26 Nov 10 2020 + +- Add `block_termination` argument to aim.Session (gorarakelyan) +- Convert infinity parameter to string in artifacts (gorarakelyan) + +## 2.0.25 Nov 9 2020 + +- Reconstruct run metadata file when running close command (gorarakelyan) + +## 2.0.24 Nov 8 2020 + +- Add SIGTERM signal handler (gorarakelyan) +- Run `track` function in a parallel thread (gorarakelyan) +- Add SDK session flush method (gorarakelyan) +- Flush aggregated metrics at a given frequency (gorarakelyan) +- Update run metadata file only on artifacts update (gorarakelyan) + +## 2.0.23 Nov 5 2020 + +- Make experiment name argument required in SDK close command (gorarakelyan) + +## 2.0.22 Nov 5 2020 + +- Add SDK `close` method to close dangling experiments (gorarakelyan) + +## 2.0.21 Nov 1 2020 + +- Resolve compatibility issues with python 3.5.0 (gorarakelyan) + +## 2.0.20 Oct 26 2020 + +- Enable pypi aim package name (gorarakelyan) + +## 2.0.19 Oct 25 2020 + +- Add PyTorch Lightning logger (gorarakelyan) +- Add TensorFlow v1 and v2 keras callbacks support (gorarakelyan) + +## 2.0.18 Oct 7 2020 + +- Add ability to run Aim UI in detached mode (gorarakelyan) +- Add ability to specify repo path when running Aim UI (gorarakelyan) + +## 2.0.17 Oct 5 2020 + +- Rename `AimDE` to `Aim UI` (gorarakelyan) + +## 2.0.16 Oct 2 2020 + +- Add ability to specify host when running AimDE (gorarakelyan) +- Disable `AimContainerCommandManager` (gorarakelyan) +- Remove `aimde` command entry point (gorarakelyan) +- Remove `de` prefix from development environment management commands (gorarakelyan) + +## 2.0.15 Sep 21 2020 + +- Set Map artifact default namespace (gorarakelyan) + +## 2.0.14 Sep 21 2020 + +- Set Metric hashable context to None if no kwarg is passed (gorarakelyan) + +## 2.0.13 Sep 21 2020 + +- Add ability to query runs by metric value (gorarakelyan) +- Add ability to query runs via SDK (gorarakelyan) + +## 2.0.12 Sep 12 2020 + +- Update Session to handle exceptions gracefully (gorarakelyan) + +## 2.0.11 Sep 11 2020 + +- Add alias to keras adapter (gorarakelyan) + +## 2.0.10 Sep 10 2020 + +- Show progress bar when pulling AimDE image (gorarakelyan) + +## 2.0.9 Sep 10 2020 + +- Add ability to start multiple sessions (gorarakelyan) +- Add Aim adapter for keras (gorarakelyan) + +## 2.0.8 Aug 26 2020 + +- Set SDK to select only unarchived runs by default (gorarakelyan) +- Add ability to archive/unarchive runs (gorarakelyan) +- Enable search by run attributes (gorarakelyan) +- Add `is not` keyword to AimQL (gorarakelyan) + +## 2.0.7 Aug 21 2020 + +- Validate Artifact values before storing (gorarakelyan) +- Add sessions to SDK (gorarakelyan) + +## 2.0.6 Aug 13 2020 + +- Add ability to retrieve metrics and traces from repo (gorarakelyan) +- Add SDK `select` method to select runs and artifacts (gorarakelyan) +- Implement search query language (gorarakelyan) + +## 2.0.5 Jul 18 2020 + +- Fix issue with PyPI reStructuredText format compatibility (gorarakelyan) + +## 2.0.4 Jul 18 2020 + +- Add ability to attach tf.summary logs to AimDE (gorarakelyan) + +## 2.0.3 Jul 8 2020 + +- Pass project path to development environment container (gorarakelyan) + +## 2.0.2 Jul 7 2020 + +- Make `epoch` argument optional for `Metric` artifact (gorarakelyan) +- Add ability to automatically commit runs after exit (gorarakelyan) +- Add `aim up` shortcut for running development environment (gorarakelyan) +- Remove first required argument(artifact name) from sdk track function (gorarakelyan) +- Add general dictionary artifact for tracking `key: value` parameters (gorarakelyan) + +## 2.0.1 Jun 24 2020 + +- Fix inconsistent DE naming (gorarakelyan) + +## 2.0.0 Jun 18 2020 + +- Tidy up aim and remove some artifacts (gorarakelyan) +- Update AimContainerCMD to open connection on custom port (gorarakelyan) +- Save passed process uuid to commit configs (gorarakelyan) +- Ability to query processes (gorarakelyan) +- Execute process and store logs into a commit of specific experiment (gorarakelyan) +- Kill running process and its children recursively (gorarakelyan) +- Keep executed processes for monitoring and management (gorarakelyan) +- Add container command handler to exec commands on the host (gorarakelyan) +- Refactor Text artifact to store sentences using protobuf and aimrecords (jamesj-jiao) +- Add ability to pass aim board port as an argument (gorarakelyan) + +## 1.2.17 May 8 2020 + +- Add config command (gorarakelyan) +- Tune artifacts: images, metric_groups, params (gorarakelyan) + +## 1.2.16 Apr 29 2020 + +- Add ability to pass numpy array as a segmentation mask (gorarakelyan) + +## 1.2.15 Apr 29 2020 + +- Add basic image list tracking (gorarakelyan) + +## 1.2.14 Apr 27 2020 + +- Optimize segmentation tracking insight to load faster (gorarakelyan) + +## 1.2.13 Apr 25 2020 + +- Remove GitHub security alert (gorarakelyan) +- Add image semantic segmentation tracking (gorarakelyan) + +## 1.2.12 Apr 20 2020 + +- Add missing init file for aim.artifacts.proto (@mike1808) + +## 1.2.11 Apr 16 2020 + +- Make epoch property optional for Metric (gorarakelyan) + +## 1.2.10 Apr 16 2020 + +- Serialize and store `Metric` records using protobuf and aimrecords (gorarakelyan) +- Create RecordWriter factory which handles artifact records saving (gorarakelyan) +- Extract artifact serialization to ArtifactWriter (mike1808) + +## 1.2.9 Mar 16 2020 + +- Alert prerequisites installation message for running board (gorarakelyan) + +## 1.2.8 Mar 15 2020 + +- Update profiler interface for keras (gorarakelyan) + +## 1.2.7 Mar 14 2020 + +- Add board pull command (gorarakelyan) +- Change board ports to 43800,1,2 (gorarakelyan) +- Add ability to profile graph output nodes (gorarakelyan) +- Remove issue with autograd inside while loop (gorarakelyan) +- Add aim board development mode (gorarakelyan) +- Update board name hash algorithm to md5 (gorarakelyan) +- Add board CLI commands: up, down and upgrade (gorarakelyan) +- Add ability to tag version as a release candidate (gorarakelyan) + +## 1.2.6 Feb 28 2020 + +- Add learning rate update tracking (gorarakelyan) + +## 1.2.5 Feb 25 2020 + +- Add autocommit feature to push command: `aim push -c [-m ]` (gorarakelyan) +- Add cli status command to list branch uncommitted artifacts (gorarakelyan) +- Add an ability to aggregate duplicated nodes within a loop (gorarakelyan) +- Remove gradient break issue when profiling output nodes (gorarakelyan) + +## 1.2.4 Feb 20 2020 + +- Enable profiler to track nodes inside loops (gorarakelyan) +- Ability to disable profiler for evaluation or inference (gorarakelyan) + +## 1.2.3 Feb 13 2020 + +- Set minimum required python version to 3.5.2 (gorarakelyan) + +## 1.2.2 Feb 13 2020 + +- Downgrade required python version (gorarakelyan) + +## 1.2.1 Feb 13 2020 + +- Edit README.md to pass reStructuredText validation on pypi (gorarakelyan) + +## 1.2.0 Feb 13 2020 + +- Make aim CLI directly accessible from main.py (gorarakelyan) +- Add disk space usage tracking (gorarakelyan) +- Add profiler support for Keras (gorarakelyan) +- Add TensorFlow graph nodes profiler (gorarakelyan) +- Add command to run aim live container mounted on aim repo (gorarakelyan) +- Update profiler to track GPU usage (gorarakelyan) +- Add machine resource usage profiler (gorarakelyan) + +## 1.1.1 Jan 14 2020 + +- Remove aim dependencies such as keras, pytorch and etc (gorarakelyan) + +## 1.1.0 Jan 12 2020 + +- Update code diff tracking to be optional (gorarakelyan) +- Add default False value to aim init function (gorarakelyan) +- Update aim repo to correctly identify cwd (gorarakelyan) +- Update push command to commit if msg argument is specified (gorarakelyan) +- Add ability to initialize repo from within the sdk (gorarakelyan) + +## 1.0.2 Jan 7 2020 + +- Remove objects dir from empty .aim branch index (gorarakelyan) + +## 1.0.1 Dec 26 2019 + +- Add cil command to print aim current version (gorarakelyan) + +## 1.0.0 Dec 25 2019 + +- Add aim version number in commit config file (gorarakelyan) +- Update push command to send username and check storage availability (gorarakelyan) +- Add hyper parameters tracking (gorarakelyan) +- Update push command to print shorter file names when pushing to remote (gorarakelyan) +- Update tracking artifacts to be saved in log format (gorarakelyan) +- Add pytorch cuda support to existing sdk artefacts (gorarakelyan) +- Add cli reset command (gorarakelyan) +- Add nested module tracking support to aim sdk (gorarakelyan) +- Add code difference tracking to aim sdk (gorarakelyan) +- Update aim push command to send commits (gorarakelyan) +- Add commit structure implementation (gorarakelyan) +- Add aim commit command synchronized with git commits (gorarakelyan) +- Add version control system factory (gorarakelyan) +- Update all insights example (gorarakelyan) +- Add model gradients tracking (gorarakelyan) +- Add model weights distribution tracking (gorarakelyan) +- Add aim correlation tracking (gorarakelyan) + +## 0.2.9 Nov 30 2019 + +- Update push tolerance when remote origin is invalid (gorarakelyan) + +## 0.2.8 Nov 30 2019 + +- Update aim auth public key search algorithm (gorarakelyan) + +## 0.2.7 Nov 14 2019 + +- Update dependencies torch and torchvision versions (sgevorg) + +## 0.2.6 Nov 5 2019 + +- Update aim track logger (gorarakelyan) + +## 0.2.5 Nov 4 2019 + +- Add branch name validation (gorarakelyan) +- Add single branch push to aim push command (gorarakelyan) + +## 0.2.4 Nov 3 2019 + +- Update aim auth print format (gorarakelyan) +- Update setup.py requirements (gorarakelyan) + +## 0.2.3 Nov 3 2019 + +- Update package requirements (gorarakelyan) + +## 0.2.2 Nov 1 2019 + +- Update package requirements (sgevorg) + +## 0.2.1 Nov 1 2019 + +- Add paramiko to required in setup.py (sgevorg) + +## 0.2.0 Nov 1 2019 + +- Update the repo to prep for open source pypi push (sgevorg) +- Add error and activity logging (sgevorg) +- Add push command robustness (gorarakelyan) +- Add cli auth command (gorarakelyan) +- Add public key authentication (gorarakelyan) +- Update push to send only branches (gorarakelyan) +- Add branching command line interface (gorarakelyan) +- Update skd interface (gorarakelyan) +- Add pytorch examples inside examples directory (gorarakelyan) +- Add model load sdk method (gorarakelyan) +- Add model checkpoint save tests (gorarakelyan) +- Update file sending protocol (gorarakelyan) +- Add model tracking (gorarakelyan) + +## 0.1.0 - Sep 23 2019 + +- Update setup py to build cython extensions (gorarakelyan) +- Update tcp client to send multiple files through one connection (gorarakelyan) +- Update tcp client to send images (gorarakelyan) +- Update sdk track functionality to support multiple metrics (gorarakelyan) +- Update push command for sending repo to a given remote (gorarakelyan) +- Add cli remote commands (gorarakelyan) +- Update cli architecture from single group of commands to multiple groups (gorarakelyan) +- Add testing env first skeleton and versions (sgevorg) +- Add dummy exporting files from .aim-test (sgevorg) +- Add description for Testing Environment (sgevorg) +- Update metadata structure and handling (sgevorg) +- Add support for seq2seq models (sgevorg) +- Update the output of doker image build to be more informative and intuitive (sgevorg) +- Update README.MD with changed Aim messaging (sgevorg) +- Remove setup.cfg file (maybe temporarily) (sgevorg) +- Update the location for docker build template files, move to data/ (sgevorg) +- Update the `docs/cli.md` for aim-deploy docs (sgevorg) +- Add docker deploy `.aim/deploy_temp/` cleanup at the end of the build (sgevorg) +- Add Docker Deploy via `aim-deploy` command (sgevorg) +- Add Docker image generate skeleton (sgevorg) +- Add AimModel.load_mode static function to parse `.aim` files (sgevorg) +- Update exporter to decouple from specifics of exporting and framework (sgevorg) +- Add model export with `.aim` extension (sgevorg) +- Remove pack/unpack of the metadata (sgevorg) +- Add pack/unpack to add metadata to model for engine processing (sgevorg) +- Add aim-deploy command configuration in cli (sgevorg) +- Add basic cli (sgevorg) +- Update setup.py for cli first version (sgevorg) +- Add initial cli specs (sgevorg) +- Add directories: the initial skeleton of the repo (sgevorg) +- Add gitignore, license file and other basics for repo (sgevorg) diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..91a57ac --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,16 @@ +cff-version: 1.2.0 +message: "If you want to cite the tool, feel free to use this (but only if you loved it 😊)" +title: "Aim" +abstract: "An easy-to-use and performant open-source ML experiment tracking tool." +date-released: 2020-06-18 +authors: + - family-names: "Arakelyan" + given-names: "Gor" + - family-names: "Soghomonyan" + given-names: "Gevorg" + - name: "The Aim team" +version: 3.9.3 +doi: 10.5281/zenodo.6536395 +license: "Apache-2.0" +url: "https://aimstack.io" +repository-code: "https://github.com/aimhubio/aim" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d639fa3 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +## Code of Conduct + +### Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +### Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +### Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +### Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +### Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at community@aimstack.io. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +### Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2f5a583 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,153 @@ +# Contributing + +Thanks for your interest in helping improve Aim! 🎉 + +## Before Contributing + +As with most projects, prior to starting to code on a bug fix or feature request, please post in the respective GitHub issue saying you want to volunteer, and then wait for a positive response. And if there is no issue for it yet, create it first. + +This helps make sure: +1. Two people aren't working on the same thing. +2. This is something Aim's maintainers believe should be implemented/fixed. +3. Any API, UI, or deeper architectural changes that need to be implemented have been fully thought through by the community together with Aim's maintainers. + +Please follow [Aim Contributor Covenant Code of Conduct](./CODE_OF_CONDUCT.md) in all your interactions with the project. + +## Governance + +This section describes governance processes we follow in developing Aim. + +### Persons of Interest + +#### Authors + +- Gev Soghomonyan ([SGevorg](https://github.com/SGevorg)) +- Gor Arakelyan ([gorarakelyan](https://github.com/gorarakelyan)) + +#### Primary Maintainers + +- Albert Torosyan ([alberttorosyan](https://github.com/alberttorosyan)) +- Karen Hambardzumyan ([mahnerak](https://github.com/mahnerak)) +- Ruben Karapetyan ([roubkar](https://github.com/roubkar)) + +#### Core Contributors + +- Arsen Alkamyan ([arsengit](https://github.com/arsengit)) +- Karo Muradyan ([KaroMourad](https://github.com/KaroMourad)) +- Mihran Vanyan ([mihran113](https://github.com/mihran113)) + +### Releases + +We release a new minor version (e.g., 3.8.0) every three to four week and patch releases on demand. The minor versions contain new features, bugfixes and also all previous bugfixes included in previous patch releases. With every release, we publish a [CHANGELOG](./CHANGELOG.md) where we list enhancements and fixes. The versioning scheme we use is [SemVer](http://semver.org/). + +## Contribution Process + +The Aim contribution process starts with filing a GitHub issue. Aim defines six categories of issues: enhancements (feature requests), bug reports, code health improvements, peformance improvements, tests, questions. + +Aim maintainers actively triage and respond to GitHub issues. In general, we recommend waiting for feedback from an Aim maintainer or community member before proceeding to implement a feature or patch. This is particularly important for significant changes, and will typically be labeled during triage with `phase / exploring`. + +After you have agreed upon an implementation strategy for your feature or patch with an Aim maintainer, the next step is to introduce your changes as a pull request against the Aim Repository. + +Once your pull request against the Aim Repository has been merged, your corresponding changes will be automatically included in the next Aim release. Every change is listed in the [Aim release notes](https://github.com/aimhubio/aim/releases) and [CHANGELOG](./CHANGELOG.md). + +Congratulations, you have just contributed to Aim. We appreciate your contribution! + +## Developing and Testing + +The majority of the Aim product areas is developed in Python/Cython. This includes the Storage, SDK, Tracking Server, CLI, API. Aim UI is a Web app mostly built with TypeScript and React. + +### Developing Storage/SDK/CLI + +Most of the backend components, including SDK, Storage, Web APIs and CLI are developed using Python/Cython. +In order to start development you must install dev requirements and the aim package itself, in editable mode. +```shell +pip install -r requirements.dev.txt +pip install -e . +``` + +Verify that Aim installed properly by running +```shell +aim version +``` +or by importing aim in python REPL +```python +import aim +``` + +#### Style Guide +Aim follows PEP8 standard for style guide and uses `flake8` as a style checker. Style checks enforced +as a check on GitHub Actions when new PR opened. + +#### Testing Requirements + +New unit-tests must be added along with the code changes. In order to setup the testing environment +```shell +cd tests +pip install -r requirements.txt +``` + +Aim python code unit-tests are located at `tests/` directory. Unit-tests are written in Python's `unittest` package style. +[Pytest](https://docs.pytest.org) is used as a test runner/discovery tool. To make sure unit-tests are not failing run +```shell +pytest tests/ +``` + +### Developing UI + +Aim UI is written in TypeScript. `npm` is required to build Aim UI and to run in DEV mode. +You can verify that `npm` is on the PATH by running `npm -v`, and +[install npm](https://www.npmjs.com/get-npm) if needed. + +#### Style Guide + +We use Prettier to autoformat code on presubmit. + +#### Launching the Development UI + +Before running the Aim UI dev server or building a distributable wheel, install npm +dependencies via: + +```shell +cd aim/web/ui +npm install +``` + +Then you can start the dev server: + +```shell +npm start +``` + +Aim UI will show logged data in at [http://localhost:3000](http://localhost:3000). + +#### Adding New Components + +To start building a new component you can run following command: + +```shell +npm run crc 'ComponentName' +``` + + +If you want to add a component inside UI kit you can run following command: + +```shell +npm run crc-kit 'ComponentName' +``` + +These command will create a folder named `ComponentName` with all the necessary files. + +## Writing Docs + +Aim documentation is built using [Sphix](https://www.sphinx-doc.org) and is hosted at +[Read the Docs](https://aimstack.readthedocs.io). +The documentation sources are located at `docs/` directory. In order to build documentation locally +run the following commands +```shell +pip install -r requirements.dev.txt +cd docs +pip install -r requirements.txt +make html +``` + +Documentation will be available at `docs/build/html/index.html` on your local machine. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..24cee6c --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019-2022 Aimhub, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..5763265 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include README.md LICENSE aim/VERSION +recursive-include aim/storage *.h *.hpp *.cpp *.cxx *.pxd *.pyx +recursive-include aim/storage/migrations * +recursive-include aim/web/migrations * + diff --git a/README.md b/README.md index 60be5d5..76bf438 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,550 @@ -# aim +
+ + + + + + + +
Drop a star to support Aim ⭐ + Join Aim discord community + +
+
+
+ +

+ An easy-to-use & supercharged open-source experiment tracker +

+ Aim logs your training runs and any AI Metadata, enables a beautiful UI to compare, observe them and an API to query them programmatically. +
+ +
+ +
+ + [![Discord Server](https://dcbadge.vercel.app/api/server/zXq2NfVdtF?compact=true&style=flat)](https://community.aimstack.io/) + [![Twitter Follow](https://img.shields.io/twitter/follow/aimstackio?style=social)](https://twitter.com/aimstackio) + [![Medium](https://img.shields.io/badge/Medium-12100E?style=flat&logo=medium&logoColor=white)](https://medium.com/aimstack) + + [![Platform Support](https://img.shields.io/badge/platform-Linux%20%7C%20macOS-blue)]() + [![PyPI - Python Version](https://img.shields.io/badge/python-%3E%3D%203.7-blue)](https://pypi.org/project/aim/) + [![PyPI Package](https://img.shields.io/pypi/v/aim?color=yellow)](https://pypi.org/project/aim/) + [![License](https://img.shields.io/badge/License-Apache%202.0-orange.svg)](https://opensource.org/licenses/Apache-2.0) + [![PyPI Downloads](https://img.shields.io/pypi/dw/aim?color=green)](https://pypi.org/project/aim/) + [![Issues](https://img.shields.io/github/issues/aimhubio/aim)](http://github.com/aimhubio/aim/issues) + +
+ +
+
+ + + +
+ +
+ +
+ SEAMLESSLY INTEGRATES WITH: +
+
+ + + + + + + + + + + + + + + + + +
+ +
+ +
+ TRUSTED BY ML TEAMS FROM: +
+
+ + + + + + + + +
+ +
+ +

+ AimStack offers enterprise support that's beyond core Aim. Contact via hello@aimstack.io e-mail. +

+ +--- + +

+ About • + Demos • + Ecosystem • + Quick Start • + Examples • + Documentation • + Community • + Blog +

+ +--- + +# ℹ️ About + +Aim is an open-source, self-hosted ML experiment tracking tool designed to handle 10,000s of training runs. + +Aim provides a performant and beautiful UI for exploring and comparing training runs. +Additionally, its SDK enables programmatic access to tracked metadata — perfect for automations and Jupyter Notebook analysis. + +

+ Aim's mission is to democratize AI dev tools 🎯 +

+ +
+ + + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + + +
Log Metadata Across Your ML Pipeline 💾Visualize & Compare Metadata via UI 📊
+
    +
  • ML experiments and any metadata tracking
    • +
  • Integration with popular ML frameworks
    • +
  • Easy migration from other experiment trackers
    • +
+ 		+
    +
  • Metadata visualization via Aim Explorers
    • +
  • Grouping and aggregation
    • +
  • Querying using Python expressions
    • +
+
Run ML Trainings Effectively ⚡Organize Your Experiments 🗂️
+
    +
  • System info and resource usage tracking
    • +
  • Real-time alerting on training progress
    • +
  • Logging and configurable notifications
    • +
+ 		+
    +
  • Detailed run information for easy debugging
    • +
  • Centralized dashboard for holistic view
    • +
  • Runs grouping with tags and experiments
+
+
+ +# 🎬 Demos + +Check out live Aim demos NOW to see it in action. + +| [Machine translation experiments](http://play.aimstack.io:10001/metrics?grouping=HQGdK9Xxy35e6sY1CYkCmk1WbWMN2AsCNfJJ3d1RJYLtrVPMoF5UpGiA6CF8bEJnfzRsKpqespf3AEuKSVrhUYvYk9MxzNGA9XZWYUf6phEg8AMbZGLRVDXnAPDuo8tueqsST1ZLizWzQwDYJWHUza6pyB2Eojt9uWqNHUdb858TqDRnCJzqiVJXKXEzFWUyvU8MckJo1qpqWWCTb4GpYN6DUJZx2GXDGR21e2xxd4m7PmNUnbA9B3apLttZoipJF6c3v7tNUKmb6irpqnNB3yc57tqYDa1XZuKfDxkMtyFdQ1x95K4jjsTVwhftEWLze35QNcxNXRCGGS9o9yEfTLG26GUX2zjPZFCjjMGU6vV7z1xRccK8MyoGrLSgAQCbvk68dTGBHpXUBvCRq8N&chart=FviZzVrt4fVQPjpCLr9sVGGrcR5etSroyqambiKpm3nTgpyv4eQxKuwNX9uN8UtKmzYUhUyTMBEANHmtbwjLApkvnYeNbxGNC6PVcoqi65m1XJnSrvgt8WiD89BapFAWRUwAGx6SWD7KZPsk3RQyysU7W7FjD3Q99NusxFGhsEfD6HXc7i8xH9KHDRGjLwh6x9VTtSp4FS8HEvpLSiiJoX7LCTi8pB7dXvrQ8G5w3jPsFz4qXYFdsVaCNL1BpFFZuiqQNkfbnM84gEq7UmiV1VzM4oS3AgQHxADG3kpBVp6eKTey9F1Swd4FcUkFA9QEPjgQgqwRGjkquZ2bdDDVLBnCh7JPvboP2kifCiZZ5MDdV9MMx6PKHp4DusWyWLXiHQYPkpGPWBiuccMUXDsuJaCWJbuABdY7CyiJMv1jdHYkjabygSxehPVyEDefWAtjBfv2vaeM1xv63jadbmpKYFxft7qmuT9HvVxiGvRgs4RQFxy8K4rtFBca3HNs1mDaaY81gy9MGXyw7BS5Fniu92jaJpsWDdg6Y3AQBLZtrpJy2obEZ4yzJaCVT7JUNPAyyCUNLck393VFLoEkaD9CU5npK5R7tj1c1G3gkMNQXnSXy5NpSj8deMmXV5qz3JKu1nq2caGQKcqjzy2gLkExdm674AMFjSg9yFjK6VqASXQ17NKtWRUvaYoxGbHDAFQaMKWKh8QLm22QA9mKT8NksLptWozbgDvafnQLNMvezLU5bvKV5o75PAWYiRB56RcYfEhzaB6YWdgL7TJicyY5rFi6Az8UZ7wqB3N5iMuZdpxhKn5KbZDxyuUMuvVt24i5LVPPmmwQtqxMoJ4aLo48a2YvDW6TAkdQjNjvn6KcEEz6GTixujb1YHhMUD8v4AepWKEwKz1ddEca1P2wLQjbpihCuaqbxeohnuZZLogJdUBojBEDgrnrrVpPBaLLEkGSpkJbtrsKUuEeBo1AF3yNgHftLbynGpobVF5DhmsmddmiA6c8vSTokJxHhjpnW8mAcNHBRtmVJCT7VkdHSAhNypM4Hivwfx5jCccG9LauKmCeRMDzHiA57TX9W6ttcPHSvUyQorARQAd2oeNY4H83hZjHh9Bt8iwKZRt4xK6hrTR8tif7hq8eURXrGH9Ys7TzykXK8FHHWvLNzNnYf3E4a9NkD43MjfKvMM1hj4Q2K8MHbmRCqrmFrHP5kim9shq6mhLPTgwha32nvnrBkfPQVPwpGTzKuwE&select=CdsQ7jVNkogQhRzQR3e28Ek39AZ4Ma2y37k5zJaf9EZmQhMjy8GtGm4LGU6dRFuAVG7mYww5xDrQAE74KHQ3Kk1e6661RmcmNALAUjtHyCmrTVBMCnBGNiuq1y7EzmxoodYHU1BV1rnoefQAw2kTBtbWi11hV1P4LcwFCcXfUWF6rpRC7ehEnUCTqUV4bkGVJPLcmk9mdmiGwa2YgmnSShNGPVGZiEi1rMVECyngSRVdqdZwAeXBGWFLfqF1KbZeCo4MTF4SSmFupJ9zLhYbuojEbopyFWHQ6xs3sq9epPeaQziLM4Js7oFYRmuFWUYdFqnZngmewXWmi7tQAgVqhiT6dMjG2eTdfgX6WuRSuoHALkh2XJhHA6GfZLUcxC5Ni9YyKuBTamtaYarbNNJJ8z15WWvuUkLpjgHdEpE2h924xFdu8aoZNuiQxYGvcndaW1BTGMXS5fTKPqYfe2n8Ky2HWPkcX3hEXtyawu1F9BndKNaXLPgsdAoFBArBZnSe28YtSmTa5LRucKVBAxakvv5MWMXchAmpaGFQbZyYUoMgQLcJd7Y96x6zSR7nhwr5Ar81BrmqYz2WFLuk7osUbwsc9HbSG6CQt8p6Vg2u7DjKaZXW8pjkPHAKrHWtHEDiJPJ5rj6VsdFm3) | [lightweight-GAN experiments](http://play.aimstack.io:10002/images?grouping=E1zQzcmtDR3wibEa1MVysTvCyZEv1T8ixkCxTWExCyMnHtX2HyiF9eszvPgfd2xdJ5TUTKGpSs1bsLVq5tHAV3uWtsZmmckn6HjNtVCMyQDJpwhiEy5tAyw&select=2NEXuD7fFoaLcwRjymjA1wLmUrGs9s3AiXcCW82C367SwJt18CAB6xzkMGowrUDuDwggE1huaPVcQJpQUsmAQx1CnGiqCUBp2jPMd5mMNPX2QKQMcmvu9ZykBNkeBvCQFPd9ERuQD2g1EjWuvyJ3H53mAZTfp94LCXvR9CUsG5ei2CjQUzfZLM6DCyUr1GPaEVnY5f1EwzicNxXuoutkBgqCqaobJ7Do4q4eHAA6ooiWU6ekS3D2sLj6qYwhVTjfGCPfbWwBiH83nFkY3fLExzdeTY2zeUHeeYikQR9S7xHbVD8WvjekdQVp8X4dNLJZxiVmEqHpPRnU3ZrYsMhE7yFAAgjJwPNUzLTt6YFrtZBcmc4rwAC2oyrqysUSEr6gzL6LcJ6yuqDGf9D5tzftHbTLDkhc8B2sCgTS&images=9vt2MvuQj2Q7jxGQYhNH6ZnWw4CsEzubFcFotuqCHfzvuruDs6pyWfhqhinD4hCiYsAURXgJbmq2L5z4vEQMbrE7iTy8XHNndPBPyuCEvRpxGwwFkukX3YGkVhNDQmUPtBagKbsMAgUASJM8hFtKboqbu9KWTModsjd4Qag7aL1KbJCzBYmZLCpKMSf6eKUTQtfwLLWbgquEx6oahAoSujV6aZ5cjsjN4JdGtPbicySpccgLDQHaQYTHCseA6sPVaEwCsoQDJAcTnjEVFFUUUW5HbPkrNgeRKb8M9pxudrweRQ3gNukLx5yizxQKrmcKU7saxLraqYUA2y5LmEQohsWGUq8sKkvGDH6oNLx2ytJsdVM5PGieENXMAaPg3KuWYXXTwixzwscdDsHSWeiXTGj1QxUKiBCnfwkZ7pZbYMCSgczSn9WpwygrKhb2znSYhn4gFzCsdjiXPPDv9LpPzkFVbsMCvk1CadqpwxTfxNmteKm7CQVViyCrvheGAk5rKpPzaBc5agyvfKpUqgRarxojnG8a4s1Y7qFT1rNVSC13C9h5fG54dDoFHxDyvej3bVTMDYsAiie3eVA3yEskyBGwApPNtjLY2H4b9jTmR3V7jnA9moFGfwMiXUjt8eoJsWTNkqBdRGSnqdva8zi5bApQaggnLebgCRpK1g8VvPrVS3ABQC8aMZJ2vibebHePWs1ahWZ2AXUUYwcuSRkiUWHwgtG9U1x6rR41UxFFNvW9rpDsU99DWzYpdgxfU75wTEPb2qeXYPxV1zVt5ixcFfA3Lvtsp5XXyfHY9FaNFeKKzAUQXPAkMWG4yH4Tp5me8Nt4puBC4pvJrboVcQdSsYhtxj2YwUjzN7Jyn9BV28dtRFPdtFUUc9pKpLvhZAD6XPDtKqrN3pG3LwYTKAiMDtC6tHvDqhQGuJGQZH5cVyTKkT48Xup4znass8tJxUJwacVQa6x2ewyd8AXCfc4j9bPQssabADmc1ho5Eghn5qe82cEcyG1okdfBCRMfmZ5EeCeKQYmoXddxM2cAwfJzCzG9bGtaMvXk3VV8TrSiRKjg3Exbftv8gx12QAzoBP9zosuULFpEAPZF1TvHJbEUmYgu9gwuRTAS3qYiywB7dsCq8wsTr7qmwt8WFFucpte8WvrkRGYy1GA7bD6uPhvS6sr1Wv259oB7Tkr5kirMo6Vdkz8ex9zVd4h2AP1J1dy8cqXaSk5B3HTZ6n1qdAMt4faLtt8SNqg4EqcvXx6r2J1czzXAPa9oSseYifvedcMyxnWkcTvno4QA6sp6zH25ubEwPAVzZZk35nNoJPasH3PgEgLafGPLCsPDD2sku5djPjfqkbDLUWMYm7BbTr7xK8v4UoTS485rPiF6VKoNQSuEnKQMT3uNRTS4EXNMjyRfUs4gk1217EhGVLhfqiZQyG4gqEhcJE3phLydLskk36PyGEbyFyvigjwvrK6boJnFpesze6Czc13HdWbWp6LHLseYujigdmdktU6EQb5KmghstmJ9gUF14JVPjYP57xtv19UT8XDuaJfwJn9z3U17ZDFnQ5zbXKSwD9ikMEd6VFo1xLBRHSmRdFSqcC96s23qWmMhheGtv6tTQAkq7CB1J1gy3skuFJXqhs1RvFWbFFUCLmHeTCtskEsQVP5Rkzat5Jn3QtSqCiRpEGc9Ykd5bWFAaqoudGcqEt993tVfVS3ZrVKAa6NDmbtAcdnfsUZxDt2muRPJDNVCBNW5k8XvevMpMsL3uCETtdutufp1VyLur2Yyx5WA8AeeFeDBxRxad3ZHbH27XdMpxWHF26hnbQAewspG1weRpVW9Ebc4Lc53RBeu8gVmTbKydrri1FHaYySZqCxht8bN4kdqSmkymmcTN3cfRN9DmzcmfKG6GbTDeCA9oXz5cVqrGXZcAiaj1oinnByW7W8GwhtK1Tzd7LG74Nu35DUdPCJXMH2ug4SEa3yXERXCaLvAHvFZAS89e7RUPpr3nTTrQLurjHSdkJ39pwEJpDcDjeWHsJSmTG1x195e6xvMmgPxAZd3Lzyk8Cxme8p1cY7FehSbTPc3zAAwi9LDGYyoQRcdbRHPLJ2W8rt9KeNfNq9moa1RVFPCPvhGuuyycT4f4QkP4Nvy4iUCaB5d8B1hcgmtg2X9Zpg6GUR32RYneQigK6S9ZYPNnaFeCNZZrwaYjkDpKMTMB6N24JC1TEAH8en3kXzf8CpLWeJpxoyB3hcCxjFHLYaovzgfGPeFBPY6ADDUcT3xkpUUEybdxE1cX7drHvBwyGqeU5g7i424tydxqufUgPY5sF9bM6mdoA3AvqDD9B3Zai71irxYXX8e6rRck4RwptJgBMX2gbotizoz9LrUwFQ2naBfJvbfEhZNCzME8a7H2YiVcq4Z6pkfbT1uMLfaixfw8nQCzVRbJAyVZgGzVbBj242LpD48R6VmxGcU5t2XkN8hZyYdBk1Uds9QyUG9VpC8ka7HjkvxBMknk6v4BjMnHnAj4ZxDUxMWEDbWw6iWD3iYWzVn3n5dzRcAqCQv3m2ZUnwuHHCTVJVZKZVyxrFP5eznpNv87RUXMfjbXypoLJFVtMoq81y82hYRFSkbAUwzhhoXBAGeBGDmDcwky2Hf7ZmfkzDLnRke916VxhTRLr8c6nXokCn8xwweuJHFeBqx7D88gpRbn5RrnH33545zyzyNpZpabQUGY3L7G3QznVw6wCS9x7FMixW2mgCeeWFhPDiz5Kz6DyyjaT413VSoRBCRakNcitYHUXqqCUPsFmZ3LTedA8jN99fYzse5LX36TSVbjnM7XmiZ8vNoH5mUsawmvG7NXbhgoyhx4rzL7t57A4g7sQg4YhGAFzEbXrh416riiPH8r52on2VEqkjNPDnybSg3cwuR6rPfMWA7YoyEAp14aStUPaKqbM9omConMxZde5o2DpjS86G5vDBY1o7F4LnBHLHRxKfqAkTPjvEdhaYY2uY6i598po9b2fAtpUGCbXnzcNrV5Vei5WkiQAqRT6whGr29PTLsAVGed71drx7BqzNiDcFJBL9dVrVoPqYLvrYVGi89MuuWuirD7CRhXWahysjrNpFf4aHXmuXS3UD7SFgkqAZzL1hrVq77K8UhGMMWLUzE9gjP6PH4xL6fJetKaRGZNpbsqDoKuBkBAk9j1nGpYMAyuo2H2AWUyj8PUgAbi1e4KPeqNqMVT85oZ9jkCggYczgNhT8gw5QsMarouMctMdbokxRfxz2xt9r2DuNmbEmq9e13Tqv94VrzR91R2o7pvH7YUFtJvcoJwR8K5jyof5SfKHT53zaBKxkLfCpPP3qR9ZCbAzVbreFKsQnCcZpd643VA9wtgKXxc375NwKj4QbnvafKNU9qc455d3S3o57mU4DFA7yHSqY1q41zySxfXYx4txL4TiqeyyTQu7KcHYbTUYRs69pkE1rWRW84N1qmisw2o7iLQPrhWkixrRDRk5toYWQg6ZDZExCyedYBGjsUAut)| +|:---:|:---:| +| | | +| Training logs of a neural translation model(from WMT'19 competition). | Training logs of 'lightweight' GAN, proposed in ICLR 2021. | + +| [FastSpeech 2 experiments](http://play.aimstack.io:10004/runs/d9e89aa7875e44b2ba85612a/audios)| [Simple MNIST](http://play.aimstack.io:10003/runs/7f083da898624a2c98e0f363/distributions) | +|:---:|:---:| +| | | +| Training logs of Microsoft's "FastSpeech 2: Fast and High-Quality End-to-End Text to Speech". | Simple MNIST training logs. | + +# 🌍 Ecosystem + +Aim is not just an experiment tracker. It's a groundwork for an ecosystem. +Check out the two most famous Aim-based tools. + +| [aimlflow](https://github.com/aimhubio/aimlflow) | [Aim-spaCy](https://github.com/aimhubio/aim-spacy) | +|:---:|:---:| +| ![aimlflow](https://user-images.githubusercontent.com/97726819/225957836-cdec88e3-4993-435a-a135-d78be3ac1635.png) | ![Aim-spaCy](https://user-images.githubusercontent.com/97726819/225957990-4edc4525-1f65-4405-b663-ce7af888bdfa.png) | +| Exploring MLflow experiments with a powerful UI | an Aim-based spaCy experiment tracker | + +# 🏁 Quick start + +Follow the steps below to get started with Aim. + +## 1. Install Aim on your training environment + +```shell +pip3 install aim +``` + +## 2. Integrate Aim with your code + +```python +from aim import Run + +# Initialize a new run +run = Run() + +# Log run parameters +run["hparams"] = { + "learning_rate": 0.001, + "batch_size": 32, +} + +# Log metrics +for i in range(10): + run.track(i, name='loss', step=i, context={ "subset":"train" }) + run.track(i, name='acc', step=i, context={ "subset":"train" }) +``` + +_See the full list of supported trackable objects(e.g. images, text, etc) [here](https://aimstack.readthedocs.io/en/latest/quick_start/supported_types.html)._ + +## 3. Run the training as usual and start Aim UI + +```shell +aim up +``` + +## Learn more + +
+ +Migrate from other tools + + +
+ +Aim has built-in converters to easily migrate logs from other tools. +These migrations cover the most common usage scenarios. +In case of custom and complex scenarios you can use Aim SDK to implement your own conversion script. + +- [TensorBoard logs converter](https://aimstack.readthedocs.io/en/latest/quick_start/convert_data.html#show-tensorboard-logs-in-aim) +- [MLFlow logs converter](https://aimstack.readthedocs.io/en/latest/quick_start/convert_data.html#show-mlflow-logs-in-aim) +- [Weights & Biases logs converter](https://aimstack.readthedocs.io/en/latest/quick_start/convert_data.html#show-weights-and-biases-logs-in-aim) + +
+ +
+ +Integrate Aim into an existing project + + +
+ +Aim easily integrates with a wide range of ML frameworks, providing built-in callbacks for most of them. + +- [Integration with Pytorch Ignite](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-pytorch-ignite) +- [Integration with Pytorch Lightning](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-pytorch-lightning) +- [Integration with Hugging Face](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-hugging-face) +- [Integration with Keras & tf.Keras](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-keras-tf-keras) +- [Integration with Keras Tuner](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-keras-tuner) +- [Integration with XGboost](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-xgboost) +- [Integration with CatBoost](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-catboost) +- [Integration with LightGBM](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-lightgbm) +- [Integration with fastai](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-fastai) +- [Integration with MXNet](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-mxnet) +- [Integration with Optuna](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-optuna) +- [Integration with PaddlePaddle](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-paddlepaddle) +- [Integration with Stable-Baselines3](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-stable-baselines3) +- [Integration with Acme](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-acme) +- [Integration with Prophet](https://aimstack.readthedocs.io/en/latest/quick_start/integrations.html#integration-with-prophet) + +
+ +
+ +Query runs programmatically via SDK + + +
+ +Aim Python SDK empowers you to query and access any piece of tracked metadata with ease. + +```python +from aim import Repo + +my_repo = Repo('/path/to/aim/repo') + +query = "metric.name == 'loss'" # Example query + +# Get collection of metrics +for run_metrics_collection in my_repo.query_metrics(query).iter_runs(): + for metric in run_metrics_collection: + # Get run params + params = metric.run[...] + # Get metric values + steps, metric_values = metric.values.sparse_numpy() +``` + +
+ +
+ +Set up a centralized tracking server + + +
+ +Aim remote tracking server allows running experiments in a multi-host environment and collect tracked data in a centralized location. + +See the docs on how to [set up the remote server](https://aimstack.readthedocs.io/en/latest/using/remote_tracking.html). + +
+ +
+ +Deploy Aim on kubernetes + + +
+ +- The official Aim docker image: https://hub.docker.com/r/aimstack/aim +- A guide on how to deploy Aim on kubernetes: https://aimstack.readthedocs.io/en/latest/using/k8s_deployment.html + + +
+ +Read the full documentation on [aimstack.readthedocs.io](https://aimstack.readthedocs.io) 📖 + +# 🆚 Comparisons to familiar tools + +
+ + TensorBoard vs Aim + + +
+ +**Training run comparison** + +Order of magnitude faster training run comparison with Aim +- The tracked params are first class citizens at Aim. You can search, group, aggregate via params - deeply explore all the tracked data (metrics, params, images) on the UI. +- With tensorboard the users are forced to record those parameters in the training run name to be able to search and compare. This causes a super-tedius comparison experience and usability issues on the UI when there are many experiments and params. **TensorBoard doesn't have features to group, aggregate the metrics** + +**Scalability** + +- Aim is built to handle 1000s of training runs - both on the backend and on the UI. +- TensorBoard becomes really slow and hard to use when a few hundred training runs are queried / compared. + +**Beloved TB visualizations to be added on Aim** + +- Embedding projector. +- Neural network visualization. + +
+ +
+ + MLflow vs Aim + + +
+ +MLFlow is an end-to-end ML Lifecycle tool. +Aim is focused on training tracking. +The main differences of Aim and MLflow are around the UI scalability and run comparison features. + +Aim and MLflow are a perfect match - check out the [aimlflow](https://github.com/aimhubio/aimlflow) - the tool that enables Aim superpowers on Mlflow. + +**Run comparison** + +- Aim treats tracked parameters as first-class citizens. Users can query runs, metrics, images and filter using the params. +- MLFlow does have a search by tracked config, but there are no grouping, aggregation, subplotting by hyparparams and other comparison features available. + +**UI Scalability** + +- Aim UI can handle several thousands of metrics at the same time smoothly with 1000s of steps. It may get shaky when you explore 1000s of metrics with 10000s of steps each. But we are constantly optimizing! +- MLflow UI becomes slow to use when there are a few hundreds of runs. + +
+ +
+ + Weights and Biases vs Aim + + +
+ +Hosted vs self-hosted + +- Weights and Biases is a hosted closed-source MLOps platform. +- Aim is self-hosted, free and open-source experiment tracking tool. + +
+ +# 🛣️ Roadmap + +## Detailed milestones + +The [Aim product roadmap](https://github.com/orgs/aimhubio/projects/3) :sparkle: + +- The `Backlog` contains the issues we are going to choose from and prioritize weekly +- The issues are mainly prioritized by the highly-requested features + +## High-level roadmap + +The high-level features we are going to work on the next few months: + +**In progress** + +- [ ] Aim SDK low-level interface +- [ ] Dashboards – customizable layouts with embedded explorers +- [ ] Ergonomic UI kit +- [ ] Text Explorer + +
+ + Next-up + + +
+ +**Aim UI** + +- Runs management + - Runs explorer – query and visualize runs data(images, audio, distributions, ...) in a central dashboard +- Explorers + - Distributions Explorer + +**SDK and Storage** + +- Scalability + - Smooth UI and SDK experience with over 10.000 runs +- Runs management + - CLI commands + - Reporting - runs summary and run details in a CLI compatible format + - Manipulations – copy, move, delete runs, params and sequences +- Cloud storage support – store runs blob(e.g. images) data on the cloud +- Artifact storage – store files, model checkpoints, and beyond + +**Integrations** + +- ML Frameworks: + - Shortlist: scikit-learn +- Resource management tools + - Shortlist: Kubeflow, Slurm +- Workflow orchestration tools + +
+ +
+ + Done + + +
+ + - [x] Live updates (Shipped: _Oct 18 2021_) + - [x] Images tracking and visualization (Start: _Oct 18 2021_, Shipped: _Nov 19 2021_) + - [x] Distributions tracking and visualization (Start: _Nov 10 2021_, Shipped: _Dec 3 2021_) + - [x] Jupyter integration (Start: _Nov 18 2021_, Shipped: _Dec 3 2021_) + - [x] Audio tracking and visualization (Start: _Dec 6 2021_, Shipped: _Dec 17 2021_) + - [x] Transcripts tracking and visualization (Start: _Dec 6 2021_, Shipped: _Dec 17 2021_) + - [x] Plotly integration (Start: _Dec 1 2021_, Shipped: _Dec 17 2021_) + - [x] Colab integration (Start: _Nov 18 2021_, Shipped: _Dec 17 2021_) + - [x] Centralized tracking server (Start: _Oct 18 2021_, Shipped: _Jan 22 2022_) + - [x] Tensorboard adaptor - visualize TensorBoard logs with Aim (Start: _Dec 17 2021_, Shipped: _Feb 3 2022_) + - [x] Track git info, env vars, CLI arguments, dependencies (Start: _Jan 17 2022_, Shipped: _Feb 3 2022_) + - [x] MLFlow adaptor (visualize MLflow logs with Aim) (Start: _Feb 14 2022_, Shipped: _Feb 22 2022_) + - [x] Activeloop Hub integration (Start: _Feb 14 2022_, Shipped: _Feb 22 2022_) + - [x] PyTorch-Ignite integration (Start: _Feb 14 2022_, Shipped: _Feb 22 2022_) + - [x] Run summary and overview info(system params, CLI args, git info, ...) (Start: _Feb 14 2022_, Shipped: _Mar 9 2022_) + - [x] Add DVC related metadata into aim run (Start: _Mar 7 2022_, Shipped: _Mar 26 2022_) + - [x] Ability to attach notes to Run from UI (Start: _Mar 7 2022_, Shipped: _Apr 29 2022_) + - [x] Fairseq integration (Start: _Mar 27 2022_, Shipped: _Mar 29 2022_) + - [x] LightGBM integration (Start: _Apr 14 2022_, Shipped: _May 17 2022_) + - [x] CatBoost integration (Start: _Apr 20 2022_, Shipped: _May 17 2022_) + - [x] Run execution details(display stdout/stderr logs) (Start: _Apr 25 2022_, Shipped: _May 17 2022_) + - [x] Long sequences(up to 5M of steps) support (Start: _Apr 25 2022_, Shipped: _Jun 22 2022_) + - [x] Figures Explorer (Start: _Mar 1 2022_, Shipped: _Aug 21 2022_) + - [x] Notify on stuck runs (Start: _Jul 22 2022_, Shipped: _Aug 21 2022_) + - [x] Integration with KerasTuner (Start: _Aug 10 2022_, Shipped: _Aug 21 2022_) + - [x] Integration with WandB (Start: _Aug 15 2022_, Shipped: _Aug 21 2022_) + - [x] Stable remote tracking server (Start: _Jun 15 2022_, Shipped: _Aug 21 2022_) + - [x] Integration with fast.ai (Start: _Aug 22 2022_, Shipped: _Oct 6 2022_) + - [x] Integration with MXNet (Start: _Sep 20 2022_, Shipped: _Oct 6 2022_) + - [x] Project overview page (Start: _Sep 1 2022_, Shipped: _Oct 6 2022_) + - [x] Remote tracking server scaling (Start: _Sep 11 2022_, Shipped: _Nov 26 2022_) + - [x] Integration with PaddlePaddle (Start: _Oct 2 2022_, Shipped: _Nov 26 2022_) + - [x] Integration with Optuna (Start: _Oct 2 2022_, Shipped: _Nov 26 2022_) + - [x] Audios Explorer (Start: _Oct 30 2022_, Shipped: _Nov 26 2022_) + - [x] Experiment page (Start: _Nov 9 2022_, Shipped: _Nov 26 2022_) + - [x] HuggingFace datasets (Start: _Dec 29 2022_, _Feb 3 2023_) + +
+ +# 👥 Community + +## Aim README badge + +Add Aim badge to your README, if you've enjoyed using Aim in your work: + +[![Aim](https://img.shields.io/badge/powered%20by-Aim-%231473E6)](https://github.com/aimhubio/aim) + +``` +[![Aim](https://img.shields.io/badge/powered%20by-Aim-%231473E6)](https://github.com/aimhubio/aim) +``` + +## Cite Aim in your papers + +In case you've found Aim helpful in your research journey, we'd be thrilled if you could acknowledge Aim's contribution: + +```bibtex +@software{Arakelyan_Aim_2020, + author = {Arakelyan, Gor and Soghomonyan, Gevorg and {The Aim team}}, + doi = {10.5281/zenodo.6536395}, + license = {Apache-2.0}, + month = {6}, + title = {{Aim}}, + url = {https://github.com/aimhubio/aim}, + version = {3.9.3}, + year = {2020} +} +``` + +## Contributing to Aim + +Considering contibuting to Aim? +To get started, please take a moment to read the [CONTRIBUTING.md](https://github.com/aimhubio/aim/blob/main/CONTRIBUTING.md) guide. + +Join Aim contributors by submitting your first pull request. Happy coding! 😊 + + + + + +Made with [contrib.rocks](https://contrib.rocks). + +## More questions? + +1. [Read the docs](https://aimstack.readthedocs.io/en/latest/) +2. [Open a feature request or report a bug](https://github.com/aimhubio/aim/issues) +3. [Join Discord community server](https://community.aimstack.io/) diff --git a/aim/VERSION b/aim/VERSION new file mode 100644 index 0000000..419f300 --- /dev/null +++ b/aim/VERSION @@ -0,0 +1 @@ +3.19.0 diff --git a/aim/__about__.py b/aim/__about__.py new file mode 100644 index 0000000..97d7dc2 --- /dev/null +++ b/aim/__about__.py @@ -0,0 +1,26 @@ +import time +import os + +here = os.path.abspath(os.path.dirname(__file__)) + +version_file = f'{here}/VERSION' + +__version__ = None +with open(version_file) as vf: + __version__ = vf.read().strip() + +_this_year = time.strftime("%Y") +__author__ = "Gev Sogomonian, Gor Arakelyan et al." +__author_email__ = "community@aimhub.io" +__license__ = "Apache-2.0" +__copyright__ = f"2019-{_this_year}, {__author__}." +__homepage__ = "https://github.com/aimhubio/aim" +__docs_url__ = "https://github.com/aimhubio/aim" +# this has to be simple string, see: https://github.com/pypa/twine/issues/522 +__docs__ = ( + "Aim is a lightweight library for ML researchers to store and retrieve AI metadata." +) +__long_docs__ = """ +""" + +__all__ = ["__author__", "__author_email__", "__copyright__", "__docs__", "__homepage__", "__license__", "__version__"] diff --git a/aim/__init__.py b/aim/__init__.py new file mode 100644 index 0000000..5f330e2 --- /dev/null +++ b/aim/__init__.py @@ -0,0 +1,8 @@ +import aimrocks +from aim.sdk import * +from aim.ext.notebook.notebook import load_ipython_extension +from aim.cli.manager.manager import run_process + +from aim.utils.deprecation import python_version_deprecation_check + +python_version_deprecation_check() diff --git a/aim/__version__.py b/aim/__version__.py new file mode 100644 index 0000000..03bad0f --- /dev/null +++ b/aim/__version__.py @@ -0,0 +1,10 @@ +import os + +here = os.path.abspath(os.path.dirname(__file__)) + +version_file = f'{here}/VERSION' + + +__version__ = None +with open(version_file) as vf: + __version__ = vf.read().strip() diff --git a/aim/acme.py b/aim/acme.py new file mode 100644 index 0000000..44884fd --- /dev/null +++ b/aim/acme.py @@ -0,0 +1,2 @@ +# Alias to SDK acme interface +from aim.sdk.adapters.acme import AimCallback, AimWriter # noqa F401 diff --git a/aim/catboost.py b/aim/catboost.py new file mode 100644 index 0000000..1b35fe1 --- /dev/null +++ b/aim/catboost.py @@ -0,0 +1,2 @@ +# Alias to SDK Catboost interface +from aim.sdk.adapters.catboost import AimLogger # noqa: F401 diff --git a/aim/cli/__init__.py b/aim/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/cli.py b/aim/cli/cli.py new file mode 100644 index 0000000..c99b828 --- /dev/null +++ b/aim/cli/cli.py @@ -0,0 +1,31 @@ +import click +from click import core + +from aim.cli.configs import * # noqa F403 +from aim.cli.init import commands as init_commands +from aim.cli.version import commands as version_commands +from aim.cli.up import commands as up_commands +from aim.cli.server import commands as server_commands +from aim.cli.reindex import commands as reindex_commands +from aim.cli.runs import commands as runs_commands +from aim.cli.convert import commands as convert_commands +from aim.cli.storage import commands as storage_commands + +core._verify_python3_env = lambda: None + + +@click.group() +@click.option('-v', '--verbose', is_flag=True) +def cli_entry_point(verbose): + if verbose: + click.echo('Verbose mode is on') + + +cli_entry_point.add_command(init_commands.init, INIT_NAME) +cli_entry_point.add_command(version_commands.version, VERSION_NAME) +cli_entry_point.add_command(up_commands.up, UP_NAME) +cli_entry_point.add_command(server_commands.server, SERVER_NAME) +cli_entry_point.add_command(reindex_commands.reindex, REINDEX_NAME) +cli_entry_point.add_command(runs_commands.runs, RUNS_NAME) +cli_entry_point.add_command(convert_commands.convert, CONVERT) +cli_entry_point.add_command(storage_commands.storage, STORAGE) diff --git a/aim/cli/configs.py b/aim/cli/configs.py new file mode 100644 index 0000000..196f33d --- /dev/null +++ b/aim/cli/configs.py @@ -0,0 +1,8 @@ +INIT_NAME = 'init' +VERSION_NAME = 'version' +UP_NAME = 'up' +REINDEX_NAME = 'reindex' +SERVER_NAME = 'server' +RUNS_NAME = 'runs' +CONVERT = 'convert' +STORAGE = 'storage' diff --git a/aim/cli/convert/__init__.py b/aim/cli/convert/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/convert/commands.py b/aim/cli/convert/commands.py new file mode 100644 index 0000000..9e9016b --- /dev/null +++ b/aim/cli/convert/commands.py @@ -0,0 +1,76 @@ +import os + +import click +from click import ClickException + +from aim.sdk.repo import Repo +from aim.sdk.utils import clean_repo_path +from aim.cli.convert.processors import ( + parse_tb_logs, + parse_mlflow_logs, + parse_wandb_logs, +) + + +@click.group() +@click.option('--repo', required=False, type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True)) +@click.pass_context +def convert(ctx, repo): + ctx.ensure_object(dict) + + repo_path = clean_repo_path(repo) or Repo.default_repo_path() + repo_inst = Repo.from_path(repo_path) + + ctx.obj['repo_inst'] = repo_inst + + +@convert.command(name='tensorboard') +@click.pass_context +@click.option('--logdir', required=True, type=click.Path(exists=True, + readable=True, + dir_okay=True, + resolve_path=True)) +@click.option('--flat', '-f', required=False, is_flag=True, default=False) +@click.option('--no-cache', required=False, is_flag=True, default=False) +def convert_tensorboard(ctx, logdir, flat, no_cache): + repo_inst = ctx.obj['repo_inst'] + parse_tb_logs(logdir, repo_inst, flat, no_cache) + + +@convert.command(name='tf') +@click.pass_context +@click.option('--logdir', required=True, type=click.Path(exists=True, + readable=True, + dir_okay=True, + resolve_path=True)) +@click.option('--flat', '-f', required=False, is_flag=True, default=False) +def convert_tensorflow(ctx, logdir, flat): + click.secho('WARN: Command \'tf\' is deprecated and will be removed in future releases,' + ' please use \'tensorboard\' instead.', fg='red') + repo_inst = ctx.obj['repo_inst'] + parse_tb_logs(logdir, repo_inst, flat) + + +@convert.command(name='mlflow') +@click.pass_context +@click.option('--tracking_uri', required=False, default=None) +@click.option('--experiment', '-e', required=False, default=None) +def convert_mlflow(ctx, tracking_uri=None, **kwargs): + repo_inst = ctx.obj['repo_inst'] + tracking_uri = tracking_uri or os.environ.get("MLFLOW_TRACKING_URI") + if not tracking_uri: + raise ClickException("MLFlow tracking URI must be provided either trough ENV or CLI.") + parse_mlflow_logs(repo_inst, tracking_uri, **kwargs) + + +@convert.command(name='wandb') +@click.pass_context +@click.option('--entity', required=True, default=None) +@click.option('--project', required=True, default=None) +@click.option('--run-id', required=False, default=None) +def convert_wandb(ctx, entity=None, project=None, **kwargs): + repo_inst = ctx.obj['repo_inst'] + parse_wandb_logs(repo_inst, entity, project, **kwargs) diff --git a/aim/cli/convert/processors/__init__.py b/aim/cli/convert/processors/__init__.py new file mode 100644 index 0000000..3f2ced4 --- /dev/null +++ b/aim/cli/convert/processors/__init__.py @@ -0,0 +1,3 @@ +from .tensorboard import parse_tb_logs +from .mlflow import parse_mlflow_logs +from .wandb import parse_wandb_logs diff --git a/aim/cli/convert/processors/mlflow.py b/aim/cli/convert/processors/mlflow.py new file mode 100644 index 0000000..4667f0f --- /dev/null +++ b/aim/cli/convert/processors/mlflow.py @@ -0,0 +1,152 @@ +import os.path +from tempfile import TemporaryDirectory + +import click + +from aim import Run, Image, Text, Audio + +IMAGE_EXTENSIONS = ('jpg', 'bmp', 'jpeg', 'png', 'gif', 'svg') +HTML_EXTENSIONS = ('html',) +TEXT_EXTENSIONS = ( + 'txt', + 'log', + 'py', + 'js', + 'yaml', + 'yml', + 'json', + 'csv', + 'tsv', + 'md', + 'rst', + 'jsonnet', +) + +# Audio is not handled in mlflow but including here just in case +AUDIO_EXTENSIONS = ( + 'flac', + 'mp3', + 'wav', +) + + +def parse_mlflow_logs(repo_inst, tracking_uri, experiment): + try: + import mlflow + except ImportError: + click.echo( + 'Could not process mlflow logs - failed to import "mlflow" module.', err=True + ) + return + + client = mlflow.tracking.client.MlflowClient(tracking_uri=tracking_uri) + + if experiment is None: + # process all experiments + experiments = client.list_experiments() + else: + try: + ex = client.get_experiment(experiment) + except mlflow.exceptions.MlflowException: + ex = client.get_experiment_by_name(experiment) + if not ex: + click.echo(f'Could not find experiment with id or name "{experiment}"', err=True) + return + experiments = (ex,) + + for ex in experiments: + runs = client.search_runs(ex.experiment_id) + for run in runs: + run_id = run.info.run_id + aim_run = Run( + repo=repo_inst, + system_tracking_interval=None, + capture_terminal_logs=False, + experiment=ex.experiment_id, + ) + aim_run['mlflow_run_id'] = run.info.run_id + aim_run['mlflow_run_name'] = run.data.tags.get("mlflow.runName") + aim_run.description = run.data.tags.get("mlflow.note.content") + + # Collect params & tags + aim_run['params'] = run.data.params + aim_run['tags'] = { + k: v for k, v in run.data.tags.items() if not k.startswith('mlflow') + } + + # Collect metrics + for key in run.data.metrics.keys(): + for m in client.get_metric_history(run_id, key): + aim_run.track(m.value, step=m.step, name=m.key) + + # Collect artifacts + __html_warning_issued = False + with TemporaryDirectory(prefix=f'mlflow_{run.info.run_id}_') as temp_path: + # click.secho(f'Downloading artifacts to {temp_path}', fg='green') + artifact_loc_stack = [None] + while artifact_loc_stack: + loc = artifact_loc_stack.pop() + artifacts = client.list_artifacts(run_id, path=loc) + + img_batch = [] + text_batch = [] + audio_batch = [] + + for file_info in artifacts: + if file_info.is_dir: + artifact_loc_stack.append(file_info.path) + continue + + downloaded_path = client.download_artifacts(run_id, file_info.path, dst_path=temp_path) + if file_info.path.endswith(HTML_EXTENSIONS): + # TODO [AP] plotly does not provide interface to load from html + # TODO [AP] need to implement html custom object? + if not __html_warning_issued: + click.secho( + 'Handler for html file types is not yet implemented.', fg='yellow' + ) + __html_warning_issued = True + continue + elif file_info.path.endswith(IMAGE_EXTENSIONS): + aim_object = Image + kwargs = dict( + image=downloaded_path, + caption=file_info.path + ) + container = img_batch + elif file_info.path.endswith(TEXT_EXTENSIONS): + with open(downloaded_path) as fh: + content = fh.read() + aim_object = Text + kwargs = dict( + text=content + ) + container = text_batch + elif file_info.path.endswith(AUDIO_EXTENSIONS): + audio_format = os.path.splitext(file_info.path)[1].lstrip('.') + aim_object = Audio + kwargs = dict( + data=downloaded_path, + caption=file_info.path, + format=audio_format + ) + container = audio_batch + else: + click.secho( + f'Unresolved or unsupported type for artifact {file_info.path}', fg='yellow' + ) + continue + + try: + item = aim_object(**kwargs) + except Exception as exc: + click.echo( + f'Could not convert artifact {file_info.path} into aim object - {exc}', err=True + ) + continue + container.append(item) + + for content_type, seq in (('image', img_batch), + ('text', text_batch), + ('audio', audio_batch)): + aim_run.track(seq, step=0, name=loc or 'root', context={'type': content_type}) diff --git a/aim/cli/convert/processors/tensorboard.py b/aim/cli/convert/processors/tensorboard.py new file mode 100644 index 0000000..3e5f240 --- /dev/null +++ b/aim/cli/convert/processors/tensorboard.py @@ -0,0 +1,262 @@ +import json +import os + +import click +from tqdm import tqdm + +from aim import Audio, Image, Run + + +def parse_tb_logs(tb_logs, repo_inst, flat=False, no_cache=False): + """ + This function scans and collects records from TB log files. + + Creates and uses cache file "tb_logs_cache" in the repo dir + to track previously processed files and values + + For more info please refer to our integration guides. + """ + + try: + # This import statement takes long to complete + import tensorflow as tf + from tensorflow.python.summary.summary_iterator import summary_iterator + from tensorboard.util import tensor_util + except ImportError: + click.echo( + 'Could not process TensorBoard logs - failed to import tensorflow module.', err=True + ) + return + + supported_plugins = ('images', 'scalars') + unsupported_plugin_noticed = False + tb_logs_cache_path = os.path.join(repo_inst.path, 'tb_logs_cache') + + if no_cache and os.path.exists(tb_logs_cache_path): + os.remove(tb_logs_cache_path) + try: + with open(tb_logs_cache_path) as FS: + tb_logs_cache = json.load(FS) + except Exception: + tb_logs_cache = {} + + def get_parent(current_path, level=0): + # level 0 is the direct parent directory + if level <= 0: + return os.path.dirname(current_path) + elif current_path in ('', '.', '/'): + return current_path + return get_parent(os.path.dirname(current_path), level - 1) + + tb_logs = os.path.abspath(tb_logs) + run_dir_candidates = set() + for root, dirs, files in os.walk(tb_logs): + for file in files: + if not file.startswith('events.out.tfevents'): + continue + + file_path = os.path.abspath(os.path.join(root, file)) + run_dir = get_parent(file_path) + + if not run_dir.startswith(tb_logs): + # it's outside tb_logs + continue + + run_dir_candidates.add(run_dir) + + def get_level(current_path): + level = -1 + while current_path.startswith(tb_logs): + current_path, _ = os.path.split(current_path) + level += 1 + return level + + def create_ndarray(tensor): + res = tensor_util.make_ndarray(tensor) + if res.dtype == "object": + return None + else: + return res + + run_dir_candidates = sorted(run_dir_candidates, key=get_level, reverse=True) + run_dir_candidates_filtered = set() + run_dir_ignored = set() + groups = set() + + for run_dir in run_dir_candidates: + if run_dir in run_dir_candidates_filtered: + # already tagged as a run dir + continue + + if run_dir in groups: + # run dir which has other run dirs inside, so we skip it + run_dir_ignored.add(run_dir) + continue + + depth = get_level(run_dir) + if depth >= 2: + if flat: + run_group_dir = get_parent(run_dir, 0) + new_run_dir = run_dir + else: + run_group_dir = get_parent(run_dir, 1) + new_run_dir = get_parent(run_dir, 0) + if new_run_dir in groups: + new_run_dir = run_dir + groups.add(run_group_dir) + elif depth == 1: + new_run_dir = run_dir + else: + continue + run_dir_candidates_filtered.add(new_run_dir) + + if run_dir_ignored: + click.echo('WARN: Found directory entries with unorganized even files!\n' + 'Please read the preparation instructions to properly process these files.\n' + 'Event files in the following directories will be ignored:', err=True) + for c, r in enumerate(run_dir_ignored, start=1): + click.echo(f'{c}: {r}', err=True) + + for path in tqdm(run_dir_candidates_filtered, + desc='Converting TensorBoard logs', + total=len(run_dir_candidates_filtered)): + + events = {} + for root, dirs, files in os.walk(path): + for file in files: + if 'events.out.tfevents' not in file: + continue + file_path = os.path.join(root, file) + if file_path == os.path.join(path, file): + entry = None + else: + entry = os.path.basename(os.path.dirname(file_path)) + events[file_path] = { + 'context': { + 'entry': entry + } + } + + if path not in tb_logs_cache: + tb_logs_cache[path] = {} + + run_cache = tb_logs_cache[path] + if run_cache: + run = Run( + run_hash=run_cache['run_hash'], + repo=repo_inst, + system_tracking_interval=None, + log_system_params=False, + capture_terminal_logs=False, + ) + else: + run = Run( + repo=repo_inst, + system_tracking_interval=None, + log_system_params=False, + capture_terminal_logs=False, + ) + run['tensorboard_logdir'] = path + run_cache.update({ + 'run_hash': run.hash, + 'events': {}, + }) + run_tb_events = run_cache['events'] + + events_to_process = [] + for event in events: + last_modified_at = os.path.getmtime(event) + try: + assert last_modified_at == run_tb_events[event]['last_modified_at'] + except (KeyError, AssertionError, RuntimeError): + # Something has changed or hasn't been processed before + events_to_process.append(event) + try: + run_tb_events[event]['last_modified_at'] = last_modified_at + except KeyError: + # Completely new event + run_tb_events[event] = { + 'last_modified_at': last_modified_at, + 'values': {}, + } + + if not events_to_process: + continue + + for event_file in tqdm(events_to_process, desc=f'Parsing logs in {path}', total=len(events_to_process)): + run_tb_log = run_tb_events[event_file] + event_context = events[event_file]['context'] + try: + for event in summary_iterator(event_file): + timestamp = event.wall_time + step = event.step + fail_count = 0 + _err_info = None + + for value in event.summary.value: + tag = value.tag + + plugin_name = value.metadata.plugin_data.plugin_name + value_id = f'{tag}_{plugin_name}' + if value_id in run_tb_log['values']: + if run_tb_log['values'][value_id]['timestamp'] >= timestamp: + # prevent previously tracked data from re-tracking upon file update + continue + + if len(plugin_name) > 0 and plugin_name not in supported_plugins: + if not unsupported_plugin_noticed: + click.echo( + 'Found unsupported plugin type in the log file. ' + 'Data for these wont be processed. ' + 'Supported plugin types are: {}'.format(', '.join(supported_plugins)), + err=True + ) + unsupported_plugin_noticed = True + continue + track_val = None + try: + if value.HasField('tensor'): + # TODO: [MV] check the case when audios are passed via tensor + if plugin_name == 'images': + tensor = value.tensor.string_val[2:] + track_val = [ + Image(tf.image.decode_image(t).numpy()) for t in tensor + ] + if len(track_val) == 1: + track_val = track_val[0] + elif plugin_name == "scalars" or plugin_name == "": + track_val = create_ndarray(value.tensor) + else: + track_val = value.tensor.float_val[0] + elif value.HasField('simple_value'): + track_val = value.simple_value + elif value.HasField('image'): + track_val = Image(tf.image.decode_image(value.image.encoded_image_string).numpy()) + elif value.HasField('audio'): + tf_audio, sample_rate = tf.audio.decode_wav(value.audio.encoded_audio_string) + track_val = Audio(tf_audio.numpy(), rate=sample_rate) + + except RuntimeError as exc: + # catch all the nasty failures + fail_count += 1 + if not _err_info: + _err_info = str(exc) + continue + + run_tb_log['values'][value_id] = { + 'step': step, + 'timestamp': timestamp + } + if track_val is not None: + run._tracker._track(track_val, timestamp, tag, step, context=event_context) + if fail_count: + click.echo(f'Failed to process {fail_count} entries. First exception: {_err_info}', err=True) + + except RuntimeError as exc: + click.echo(f'Failed to read log file {event_file} - {exc}', err=True) + + # refresh cache + with open(tb_logs_cache_path, 'w') as FS: + json.dump(tb_logs_cache, FS) + + click.echo('TensorBoard logs conversion complete!') diff --git a/aim/cli/convert/processors/wandb.py b/aim/cli/convert/processors/wandb.py new file mode 100644 index 0000000..23c81bb --- /dev/null +++ b/aim/cli/convert/processors/wandb.py @@ -0,0 +1,182 @@ +from pathlib import Path +import re +from tempfile import TemporaryDirectory + +import click +from tqdm import tqdm + +from aim import Run +from aim.ext.resource.log import LogLine +from aim.ext.resource.configs import AIM_RESOURCE_METRIC_PREFIX + + +def parse_wandb_logs(repo_inst, entity, project, run_id): + try: + import wandb + except ImportError: + click.echo("Could not process wandb logs - failed to import 'wandb' module.", err=True) + return + + client = wandb.Api() + + if run_id is None: + # process all runs + runs = client.runs(entity + "/" + project) + else: + try: + # get the run by run_id + run = client.run(f"{entity}/{project}/{run_id}") + except Exception: + click.echo(f"Could not find run '{entity}/{project}/{run_id}'", err=True) + return + runs = (run,) + + for run in tqdm(runs, desc="Converting wandb logs"): + if not run.config.items(): + continue + aim_run = Run( + repo=repo_inst, + system_tracking_interval=None, + capture_terminal_logs=False, + experiment=project + ) + aim_run['wandb_run_id'] = run.id + aim_run['wandb_run_name'] = run.name + aim_run.description = run.notes + + with TemporaryDirectory() as tmpdirname: + # Collect console output logs + console_log_filename = 'output.log' + console_log_file = run.file(console_log_filename) + try: + # Even though the file does not exist, a file object will be returned in zero-sized. + if console_log_file.size: + console_log_file.download(root=tmpdirname) + with open(Path(tmpdirname) / console_log_filename) as f: + [aim_run.track(LogLine(line), name='logs', step=i) for i, line in enumerate(f.readlines())] + except Exception: + click.echo("Failed to track console output log.", err=True) + + # TODO: Collect media files, possibly? + + # Collect params & tags + aim_run['params'] = run.config + for tag in run.tags: + aim_run.add_tag(tag) + + keys = [key for key in run.history(stream='default').keys() + if not key.startswith('_')] + + # Collect metrics + for record in run.scan_history(): + step = record.get('_step') + epoch = record.get('epoch') + timestamp = record.get('_timestamp') + for key in keys: + value = record.get(key) + if value is None: + continue + try: + tag, name = key.rsplit("/", 1) + if "train" in tag: + context = {'tag': tag, 'subset': 'train'} + elif "val" in tag: + context = {'tag': tag, 'subset': 'val'} + elif "test" in tag: + context = {'tag': tag, 'subset': 'test'} + else: + context = {'tag': tag} + except ValueError: + name, context = key, {} + try: + if timestamp: + aim_run._tracker._track(value, track_time=timestamp, name=name, + step=step, epoch=epoch, context=context) + else: + aim_run.track(value, name=name, step=step, epoch=epoch, context=context) + except ValueError: + click.echo(f"Type '{type(value).__name__}': artifacts are not supported yet.", err=True) + + # Collect system logs + # NOTE: In 'system' logs, collecting sampled history cannot be avoided. (default 'samples' == 500) + # TODO: async history fetching for better performance + for record in run.history(stream='system', pandas=False, samples=1e3): + timestamp = record.get('_timestamp') + for key in record: + if key.startswith('_'): # Including '_runtime', '_timestamp', '_wandb' + continue + + value = record.get(key) + if value is None: + continue + + name, context = _normalize_system_metric_key(key) + if name is None: + continue + + try: + if timestamp: + aim_run._tracker._track(value, track_time=timestamp, + name=f'{AIM_RESOURCE_METRIC_PREFIX}{name}', context=context) + else: + aim_run.track(value, name=f'{AIM_RESOURCE_METRIC_PREFIX}{name}', context=context) + except ValueError: + click.echo(f"Type '{type(value).__name__}': artifacts are not supported yet.", err=True) + + +def _normalize_system_metric_key(key): + # Remap names for being categorized as `System` in aim ui: `aim/web/ui/src/config/systemMetrics/systemMetrics.ts`` + # {value is None} means not supported yet + SYSTEM_METRICS_NAME_MAP = { + '': { + 'cpu': 'cpu', + 'disk': 'disk_percent', + 'memory': 'memory_percent', + 'network.recv': None, + 'network.sent': None, + }, + 'gpu': { + 'gpu': 'gpu', + 'memory': 'gpu_memory_percent', + 'memoryAllocated': None, + 'powerPercent': None, + 'powerWatts': 'gpu_power_watts', + 'temp': 'gpu_temp', + }, + 'proc': { + 'cpu.threads': None, + 'memory.availableMB': None, + 'memory.percent': 'p_memory_percent', + 'momory.rssMB': None, + } + } + + name = re.sub(r'^system\.', '', key) + gpu_idx_pattern = re.compile(r'^[0-9]+\.') + + # Triage & Remap name for aim ui + if name.startswith('gpu'): + name = re.sub(r'^gpu\.', '', name) + + # Cut & paste gpu idx from name to context + gpu_idx_match = gpu_idx_pattern.search(name) + if gpu_idx_match: + gpu_idx_str = gpu_idx_match.group() + name = name[len(gpu_idx_str):] + gpu_idx = int(gpu_idx_str.rstrip('.')) + context = {'gpu': gpu_idx, 'tag': 'system', 'subset': 'gpu'} + else: + context = {'gpu': 'no_idx', 'tag': 'system', 'subset': 'gpu'} + + normalized_name = SYSTEM_METRICS_NAME_MAP['gpu'].get(name) + + elif name.startswith('proc'): + name = re.sub(r'^proc\.', '', name) + normalized_name = SYSTEM_METRICS_NAME_MAP['proc'].get(name) + context = {'tag': 'system', 'subset': 'proc'} + + else: + normalized_name = SYSTEM_METRICS_NAME_MAP[''].get(name) + context = {'tag': 'system'} + + return normalized_name, context diff --git a/aim/cli/init/__init__.py b/aim/cli/init/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/init/commands.py b/aim/cli/init/commands.py new file mode 100644 index 0000000..54f32af --- /dev/null +++ b/aim/cli/init/commands.py @@ -0,0 +1,38 @@ +import click +import os + +from aim.sdk.repo import Repo +from aim.sdk.utils import clean_repo_path + + +@click.command() +@click.option('--repo', required=False, type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True)) +@click.option('-y', '--yes', is_flag=True, help='Automatically confirm prompt') +def init(repo, yes): + """ + Initializes new repository in the --repo directory. + Initializes new repository in the current working directory if --repo argument is not provided: + - Creates .aim directory & runs upgrades for structured DB + """ + repo_path = clean_repo_path(repo) or os.getcwd() + re_init = False + if Repo.exists(repo_path): + if yes: + re_init = True + else: + re_init = click.confirm('Aim repository is already initialized. ' + 'Do you want to re-initialize to empty Aim repository?') + if not re_init: + return + # Clear old repo + Repo.rm(repo_path) + + repo = Repo.from_path(repo_path, init=True) + if re_init: + click.echo( + 'Re-initialized empty Aim repository at {}'.format(repo.root_path)) + else: + click.echo('Initialized a new Aim repository at {}'.format(repo.root_path)) diff --git a/aim/cli/manager/__init__.py b/aim/cli/manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/manager/manager.py b/aim/cli/manager/manager.py new file mode 100644 index 0000000..dd7d92e --- /dev/null +++ b/aim/cli/manager/manager.py @@ -0,0 +1,128 @@ +import subprocess +import time +import enum + +from aim.cli.configs import VERSION_NAME, UP_NAME +from aim.__version__ import __version__ + +# Error message prefix for aim commands +ERROR_MSG_PREFIX = 'Error:' + + +class ManagerActionStatuses(enum.Enum): + Failed = 1, + Succeed = 2, + + +class ManagerActionResult: + """ + Object returned by manager action + If status is ManagerActionStatuses.Failed the info dict should have a message property + If status is ManagerActionStatuses.Succeed the info dict should have required properties for the specific action + @TODO add type checking for info fields + """ + def __init__(self, status: ManagerActionStatuses, info: dict = None): + self.status = status + self.info = info + + +def run_up(args): + def check_startup_success(): + import requests + server_path = 'http://{}:{}{}'.format(args['--host'], args['--port'], args['--base-path']) + status_api = f'{server_path}/api/projects/status' + retry_count = 5 + sleep_interval = 1 + for _ in range(retry_count): + try: + response = requests.get(status_api) + if response.status_code == 200: + return True + except Exception: + pass + sleep_interval += 1 + time.sleep(sleep_interval) + + return False + + args_list = ['--log-level=error'] + for p in args.keys(): + if p != '--proxy-url': + args_list.append(p + '=' + args[p]) + + child_process = subprocess.Popen( + ['aim', UP_NAME] + args_list + ['--force-init'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE + ) + # Runs `aim up ` command + info = { + 'port': args['--port'], + 'host': 'http://' + args['--host'] + } + + if check_startup_success(): + return ManagerActionResult( + ManagerActionStatuses.Succeed, + info + ) + + for line in child_process.stderr: + if ERROR_MSG_PREFIX in line.decode(): + return ManagerActionResult( + ManagerActionStatuses.Failed, + {'message': line.decode()} + ) + + return ManagerActionResult( + ManagerActionStatuses.Failed, + { + 'message': '\nPerhaps this is a bug from aim side.' + '\nPlease open an issue https://github.com/aimhubio/aim/issues.' + } + ) + + +def run_version(args): + """Aim cli command runner to get aim current version + Returns: + the current version of aim + """ + return ManagerActionResult( + ManagerActionStatuses.Succeed, + {'version': __version__} + ) + + +# command runners dict +COMMANDS = { + UP_NAME: run_up, + VERSION_NAME: run_version, + # Perhaps it will need to implement the help command as well +} + + +def __get_command_runner(command): + """Returns the runner function of the specified command""" + return COMMANDS[command] + + +def validate_command(command): + """Validate command existence + """ + return command in (VERSION_NAME, UP_NAME) + + +def run_process(command, args): + """ + Returns None if command is not exists + call the corresponding runner function to execute aim command `aim ` + """ + if validate_command(command) is not True: + return ManagerActionResult( + ManagerActionStatuses.Failed, + {'message': 'Invalid operation'} + ) + + run = __get_command_runner(command) + return run(args) diff --git a/aim/cli/reindex/__init__.py b/aim/cli/reindex/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/reindex/commands.py b/aim/cli/reindex/commands.py new file mode 100644 index 0000000..b580924 --- /dev/null +++ b/aim/cli/reindex/commands.py @@ -0,0 +1,18 @@ +import click + + +@click.command() +@click.option('--repo', required=False, type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True)) +@click.option('--finalize-only', required=False, is_flag=True, default=False) +def reindex(repo, finalize_only): + """ + Process runs left in 'in progress' state. + """ + from aim.utils.deprecation import deprecation_warning + + deprecation_warning(remove_version='3.16', msg='`aim reindex` is deprecated! ' + 'Use `aim runs close` command instead.') + return diff --git a/aim/cli/runs/__init__.py b/aim/cli/runs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/runs/commands.py b/aim/cli/runs/commands.py new file mode 100644 index 0000000..e91e93f --- /dev/null +++ b/aim/cli/runs/commands.py @@ -0,0 +1,163 @@ +import click +import os +import tqdm + +from multiprocessing.pool import ThreadPool +from psutil import cpu_count + +from aim.cli.runs.utils import match_runs, make_zip_archive, upload_repo_runs +from aim.sdk.repo import Repo + + +@click.group() +@click.option('--repo', required=False, + default=os.getcwd(), + type=str) +@click.pass_context +def runs(ctx, repo): + """Manage runs in aim repository.""" + ctx.ensure_object(dict) + ctx.obj['repo'] = repo + + +@runs.command(name='ls') +@click.pass_context +def list_runs(ctx): + """List Runs available in Repo.""" + repo_path = ctx.obj['repo'] + if not Repo.is_remote_path(repo_path): + if not Repo.exists(repo_path): + click.echo(f'\'{repo_path}\' is not a valid aim repo.') + exit(1) + + repo = Repo.from_path(repo_path) + run_hashes = repo.list_all_runs() + + click.echo('\t'.join(run_hashes)) + click.echo(f'Total {len(run_hashes)} runs.') + + +@runs.command(name='rm') +@click.argument('hashes', nargs=-1, type=str) +@click.pass_context +@click.option('-y', '--yes', is_flag=True, help='Automatically confirm prompt') +def remove_runs(ctx, hashes, yes): + """Remove Run data for given run hashes.""" + if len(hashes) == 0: + click.echo('Please specify at least one Run to delete.') + exit(1) + repo_path = ctx.obj['repo'] + repo = Repo.from_path(repo_path) + + matched_hashes = match_runs(repo, hashes) + if yes: + confirmed = True + else: + confirmed = click.confirm(f'This command will permanently delete {len(matched_hashes)} runs from aim repo ' + f'located at \'{repo_path}\'. Do you want to proceed?') + if not confirmed: + return + + success, remaining_runs = repo.delete_runs(matched_hashes) + if success: + click.echo(f'Successfully deleted {len(matched_hashes)} runs.') + else: + click.echo('Something went wrong while deleting runs. Remaining runs are:', err=True) + click.secho('\t'.join(remaining_runs), fg='yellow') + + +@runs.command(name='cp') +@click.option('--destination', required=True, type=str) +@click.argument('hashes', nargs=-1, type=str) +@click.pass_context +def copy_runs(ctx, destination, hashes): + """Copy Run data for given run hashes to destination Repo.""" + if len(hashes) == 0: + click.echo('Please specify at least one Run to copy.') + exit(1) + source = ctx.obj['repo'] + source_repo = Repo.from_path(source) + destination_repo = Repo.from_path(destination) + + matched_hashes = match_runs(source_repo, hashes) + success, remaining_runs = source_repo.copy_runs(matched_hashes, destination_repo) + if success: + click.echo(f'Successfully copied {len(matched_hashes)} runs.') + else: + click.echo('Something went wrong while copying runs. Remaining runs are:', err=True) + click.secho('\t'.join(remaining_runs), fg='yellow') + + +@runs.command(name='mv') +@click.option('--destination', required=True, + type=str) +@click.argument('hashes', nargs=-1, type=str) +@click.pass_context +def move_runs(ctx, destination, hashes): + """Move Run data for given run hashes to destination Repo.""" + if len(hashes) == 0: + click.echo('Please specify at least one Run to move.') + exit(1) + source = ctx.obj['repo'] + source_repo = Repo.from_path(source) + destination_repo = Repo.from_path(destination) + + matched_hashes = match_runs(source_repo, hashes) + + success, remaining_runs = source_repo.move_runs(matched_hashes, destination_repo) + if success: + click.echo(f'Successfully moved {len(matched_hashes)} runs.') + else: + click.echo('Something went wrong while moving runs. Remaining runs are:', err=True) + click.secho('\t'.join(remaining_runs), fg='yellow') + + +@runs.command(name='upload') +@click.argument('bucket', nargs=1, type=str) +@click.pass_context +def upload_runs(ctx, bucket): + """Upload Repo backup to the given S3 bucket.""" + repo_path = ctx.obj['repo'] + if not Repo.exists(repo_path): + click.echo(f'\'{repo_path}\' is not a valid aim repo.') + exit(1) + + zip_buffer = make_zip_archive(repo_path) + zip_buffer.seek(0) + + success, uploaded_zip_file_name = upload_repo_runs(zip_buffer, bucket) + if success: + click.echo(f'Successfully uploaded runs in {uploaded_zip_file_name}.') + else: + click.echo(f'The storage backup failed because of the following error: {uploaded_zip_file_name}.') + + +@runs.command(name='close') +@click.argument('hashes', nargs=-1, type=str) +@click.pass_context +@click.option('-y', '--yes', is_flag=True, help='Automatically confirm prompt') +def close_runs(ctx, hashes, yes): + """Close failed/stalled Runs.""" + repo_path = ctx.obj['repo'] + repo = Repo.from_path(repo_path) + + if len(hashes) == 0: + click.echo('Please specify at least one Run to close.') + exit(1) + + click.secho(f'This command will forcefully close {len(hashes)} Runs from Aim Repo \'{repo_path}\'. ' + f'Please make sure Runs are not active. Data corruption may occur otherwise.') + if yes: + confirmed = True + else: + confirmed = click.confirm('Do you want to proceed?') + if not confirmed: + return + + pool = ThreadPool(cpu_count(logical=False)) + + for _ in tqdm.tqdm( + pool.imap_unordered(repo._close_run, hashes), + desc='Closing runs', + total=len(hashes)): + pass diff --git a/aim/cli/runs/utils.py b/aim/cli/runs/utils.py new file mode 100644 index 0000000..d0a32e0 --- /dev/null +++ b/aim/cli/runs/utils.py @@ -0,0 +1,68 @@ +import fnmatch +import os + +from typing import List, Tuple, TYPE_CHECKING +import io +import zipfile +from datetime import datetime + +if TYPE_CHECKING: + from aim.sdk.repo import Repo + + +def match_runs(repo: 'Repo', hashes: List[str], lookup_dir: str = None) -> List[str]: + matched_hashes = set() + all_run_hashes = None + for run_hash in hashes: + if '*' in run_hash: + expr = run_hash # for the sake of readability + # avoiding multiple or unnecessary list_runs() calls + if not all_run_hashes: + all_run_hashes = repo.list_all_runs() + if expr == '*': + return all_run_hashes + # update the matches set with current expression matches + matched_hashes.update(fnmatch.filter(all_run_hashes, expr)) + else: + matched_hashes.add(run_hash) + + return list(matched_hashes) + + +def make_zip_archive(repo_path: str) -> io.BytesIO: + aim_dir = os.path.join(repo_path, '.aim') + zip_buf = io.BytesIO() + zipf = zipfile.ZipFile(zip_buf, 'w', zipfile.ZIP_DEFLATED) + len_dir_path = len(aim_dir) + for root, _, files in os.walk(aim_dir): + for file in files: + file_path = os.path.join(root, file) + zipf.write(file_path, file_path[len_dir_path:]) + zipf.close() + return zip_buf + + +def upload_repo_runs(buffer: io.BytesIO, bucket_name: str) -> Tuple[bool, str]: + try: + import boto3 + except ImportError: + raise RuntimeError( + 'This command requires \'boto3\' to be installed. ' + 'Please install it with command: \n pip install boto3' + ) + + try: + s3_client = boto3.client('s3') + buckets = s3_client.list_buckets() + bucket_names = [] + for bucket in buckets['Buckets']: + bucket_names.append(bucket['Name']) + + if bucket_name not in bucket_names: + s3_client.create_bucket(Bucket=bucket_name) + + key = f'aim-{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}.zip' + s3_client.upload_fileobj(buffer, bucket_name, key) + return True, key + except Exception as e: + return False, e diff --git a/aim/cli/server/__init__.py b/aim/cli/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/server/commands.py b/aim/cli/server/commands.py new file mode 100644 index 0000000..5f6376f --- /dev/null +++ b/aim/cli/server/commands.py @@ -0,0 +1,96 @@ +import os +import click + +from aim.sdk.repo import Repo +from aim.sdk.utils import clean_repo_path +from aim.cli.utils import ( + ShellCommandException, + build_uvicorn_command, + exec_cmd, + get_free_port_num, + get_repo_instance, + set_log_level +) +from aim.ext.transport.config import ( + AIM_SERVER_DEFAULT_PORT, + AIM_SERVER_DEFAULT_HOST, + AIM_SERVER_MOUNTED_REPO_PATH, + AIM_SERVER_BASE_PATH +) +from aim.web.configs import AIM_ENV_MODE_KEY + + +@click.command('server') +@click.option('-h', '--host', default=AIM_SERVER_DEFAULT_HOST, type=str) +@click.option('-p', '--port', default=AIM_SERVER_DEFAULT_PORT, type=int) +@click.option('--repo', required=False, default=os.getcwd(), type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True)) +@click.option('--ssl-keyfile', required=False, type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True)) +@click.option('--ssl-certfile', required=False, type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True)) +@click.option('--base-path', required=False, default='', type=str) +@click.option('--log-level', required=False, default='', type=str) +@click.option('--dev', is_flag=True, default=False) +@click.option('-y', '--yes', is_flag=True, help='Automatically confirm prompt') +def server(host, port, + repo, ssl_keyfile, ssl_certfile, + base_path, log_level, dev, yes): + """ + Starts the Aim remote tracking server for real-time logging. + + The Aim tracking server facilitates real-time logging of experiments + from remote locations. This command launches the server with specified + configurations, including host, port, and associated repository. + + Like the UI, the server can also run in production or development mode. + """ + if dev: + os.environ[AIM_ENV_MODE_KEY] = 'dev' + log_level = log_level or 'debug' + else: + os.environ[AIM_ENV_MODE_KEY] = 'prod' + + if log_level: + set_log_level(log_level) + + if base_path: + if base_path.endswith('/'): + base_path = base_path[:-1] + if not base_path.startswith('/'): + base_path = f'/{base_path}' + os.environ[AIM_SERVER_BASE_PATH] = base_path + + if port == 0: + try: + port = get_free_port_num() + except Exception: + pass + + repo_path = clean_repo_path(repo) or Repo.default_repo_path() + repo_inst = get_repo_instance(repo_path, yes) + + if not repo_inst: + return + + os.environ[AIM_SERVER_MOUNTED_REPO_PATH] = repo_inst.path + + click.secho('Running Aim Server on repo `{}`'.format(repo), fg='yellow') + click.echo('Server is mounted on aim://{}:{}'.format(host, port), err=True) + click.echo('Press Ctrl+C to exit') + + try: + cmd = build_uvicorn_command('aim.ext.transport.run:app', + host=host, port=port, + ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile, log_level=log_level) + exec_cmd(cmd, stream_output=True) + except ShellCommandException: + click.echo('Failed to run Aim Tracking Server. ' + 'Please see the logs above for details.') + exit(1) diff --git a/aim/cli/storage/__init__.py b/aim/cli/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/storage/commands.py b/aim/cli/storage/commands.py new file mode 100644 index 0000000..3dded26 --- /dev/null +++ b/aim/cli/storage/commands.py @@ -0,0 +1,133 @@ +import click +import os +from tqdm import tqdm + +from aim.cli.runs.utils import match_runs + +from aim.sdk.maintenance_run import MaintenanceRun as Run +from aim.sdk.utils import backup_run, restore_run_backup +from aim.sdk.repo import Repo +from aim.sdk.index_manager import RepoIndexManager + + +@click.group() +@click.option('--repo', required=False, + default=os.getcwd(), + type=str) +@click.pass_context +def storage(ctx, repo): + """Manage aim repository data & format updates.""" + ctx.ensure_object(dict) + ctx.obj['repo'] = repo + + +@storage.group() +@click.pass_context +def upgrade(ctx): + """Update Runs data for given run hashes to use new format.""" + pass + + +@upgrade.command(name='3.11+') +@click.argument('hashes', nargs=-1, type=str) +@click.pass_context +@click.option('-y', '--yes', is_flag=True, help='Automatically confirm prompt') +def to_3_11(ctx, hashes, yes): + """Optimize Runs Metrics data for read access.""" + if len(hashes) == 0: + click.echo('Please specify at least one Run to update.') + exit(1) + repo_path = ctx.obj['repo'] + repo = Repo.from_path(repo_path) + + matched_hashes = match_runs(repo, hashes) + remaining_runs = [] + if yes: + confirmed = True + else: + confirmed = click.confirm(f'This command will optimize the metrics data for {len(matched_hashes)} ' + f'runs from aim repo located at \'{repo_path}\'. This process might take a while. ' + f'Do you want to proceed?') + if not confirmed: + return + + index_manager = RepoIndexManager.get_index_manager(repo) + for run_hash in tqdm(matched_hashes): + try: + run = Run(run_hash, repo=repo) + if run.check_metrics_version(): + backup_run(run) + run.update_metrics() + index_manager.index(run_hash) + else: + click.echo(f'Run {run.hash} is already up to date. Skipping') + except Exception: + remaining_runs.append(run_hash) + + if not remaining_runs: + click.echo('Finished optimizing metric data.') + else: + click.echo('Finished optimizing metric data. The following runs were skipped:') + click.secho(' '.join(remaining_runs), fg='yellow') + click.echo('In case of any issues the following command can be used to restore data:') + click.secho(f'aim storage --repo {repo.root_path} restore \'*\'', fg='yellow') + + +@storage.command(name='restore') +@click.argument('hashes', nargs=-1, type=str) +@click.pass_context +@click.option('-y', '--yes', is_flag=True, help='Automatically confirm prompt') +def restore_runs(ctx, hashes, yes): + """Rollback Runs data for given run hashes to the previous metric format. """ + if len(hashes) == 0: + click.echo('Please specify at least one Run to delete.') + exit(1) + repo_path = ctx.obj['repo'] + repo = Repo.from_path(repo_path) + + matched_hashes = match_runs(repo, hashes, lookup_dir='bcp') + if yes: + confirmed = True + else: + confirmed = click.confirm(f'This command will restore {len(matched_hashes)} runs from aim repo ' + f'located at \'{repo_path}\'. Do you want to proceed?') + if not confirmed: + return + + remaining_runs = [] + index_manager = RepoIndexManager.get_index_manager(repo) + for run_hash in tqdm(matched_hashes): + try: + restore_run_backup(repo, run_hash) + index_manager.index(run_hash) + except Exception as e: + click.echo(f'Error while trying to restore run \'{run_hash}\'. {str(e)}.', err=True) + remaining_runs.append(run_hash) + + if not remaining_runs: + click.echo(f'Successfully restored {len(matched_hashes)} runs.') + else: + click.echo('Something went wrong while restoring runs. Remaining runs are:', err=True) + click.secho('\t'.join(remaining_runs), fg='yellow') + + +@storage.command(name='prune') +@click.pass_context +def prune(ctx): + """Remove dangling/orphan params/sequences with no referring runs.""" + + repo_path = ctx.obj['repo'] + repo = Repo.from_path(repo_path) + repo.prune() + + +@storage.command('reindex') +@click.option('--finalize-only', required=False, is_flag=True, default=False) +@click.pass_context +def reindex(ctx, finalize_only): + """ Process runs left in 'in progress' state. """ + from aim.utils.deprecation import deprecation_warning + + deprecation_warning(remove_version='3.16', msg='`aim storage reindex` is deprecated! ' + 'Use `aim runs close` command instead.') + return diff --git a/aim/cli/up/__init__.py b/aim/cli/up/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/up/commands.py b/aim/cli/up/commands.py new file mode 100644 index 0000000..acdc3e4 --- /dev/null +++ b/aim/cli/up/commands.py @@ -0,0 +1,127 @@ +import os +import click + +from aim.cli.utils import ( + ShellCommandException, + build_db_upgrade_command, + build_uvicorn_command, + exec_cmd, + get_free_port_num, + get_repo_instance, + set_log_level +) +from aim.web.configs import ( + AIM_ENV_MODE_KEY, + AIM_TF_LOGS_PATH_KEY, + AIM_UI_BASE_PATH, + AIM_UI_DEFAULT_HOST, + AIM_UI_DEFAULT_PORT, + AIM_UI_MOUNTED_REPO_PATH, + AIM_PROXY_URL, + AIM_PROFILER_KEY +) +from aim.sdk.repo import Repo +from aim.sdk.utils import clean_repo_path + + +@click.command() +@click.option('-h', '--host', default=AIM_UI_DEFAULT_HOST, type=str) +@click.option('-p', '--port', default=AIM_UI_DEFAULT_PORT, type=int) +@click.option('-w', '--workers', default=1, type=int) +@click.option('--uds', required=False, type=click.Path(exists=False, + file_okay=True, + dir_okay=False, + readable=True)) +@click.option('--repo', required=False, type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True)) +@click.option('--tf_logs', type=click.Path(exists=True, readable=True)) +@click.option('--dev', is_flag=True, default=False) +@click.option('--ssl-keyfile', required=False, type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True)) +@click.option('--ssl-certfile', required=False, type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True)) +@click.option('--base-path', required=False, default='', type=str) +@click.option('--profiler', is_flag=True, default=False) +@click.option('--log-level', required=False, default='', type=str) +@click.option('-y', '--yes', is_flag=True, help='Automatically confirm prompt') +def up(dev, host, port, workers, uds, + repo, tf_logs, + ssl_keyfile, ssl_certfile, + base_path, profiler, + log_level, yes): + if dev: + os.environ[AIM_ENV_MODE_KEY] = 'dev' + log_level = log_level or 'debug' + else: + os.environ[AIM_ENV_MODE_KEY] = 'prod' + + if log_level: + set_log_level(log_level) + + if base_path: + # process `base_path` as ui requires leading slash + if base_path.endswith('/'): + base_path = base_path[:-1] + if base_path and not base_path.startswith('/'): + base_path = f'/{base_path}' + os.environ[AIM_UI_BASE_PATH] = base_path + + repo_path = clean_repo_path(repo) or Repo.default_repo_path() + repo_inst = get_repo_instance(repo_path, yes) + + if not repo_inst: + return + + os.environ[AIM_UI_MOUNTED_REPO_PATH] = repo_inst.path + + if tf_logs: + os.environ[AIM_TF_LOGS_PATH_KEY] = tf_logs + + try: + db_cmd = build_db_upgrade_command() + exec_cmd(db_cmd, stream_output=True) + except ShellCommandException: + click.echo('Failed to initialize Aim DB. ' + 'Please see the logs above for details.') + return + + if port == 0: + try: + port = get_free_port_num() + except Exception: + pass + + click.echo(click.style('Running Aim UI on repo `{}`'.format(repo_inst), fg='yellow')) + + if uds: + click.echo('Aim UI running on {}'.format(uds)) + else: + scheme = 'https' if ssl_keyfile or ssl_certfile else 'http' + click.echo('Open {}://{}:{}{}'.format(scheme, host, port, base_path), err=True) + + proxy_url = os.environ.get(AIM_PROXY_URL) + if proxy_url: + click.echo(f'Proxy {proxy_url}{base_path}/') + + click.echo('Press Ctrl+C to exit') + + if profiler: + os.environ[AIM_PROFILER_KEY] = '1' + + try: + server_cmd = build_uvicorn_command( + 'aim.web.run:app', + host=host, port=port, + workers=workers, uds=uds, + ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile, + log_level=log_level) + exec_cmd(server_cmd, stream_output=True) + except ShellCommandException: + click.echo('Failed to run Aim UI. Please see the logs above for details.') + return diff --git a/aim/cli/utils.py b/aim/cli/utils.py new file mode 100644 index 0000000..b3f95f9 --- /dev/null +++ b/aim/cli/utils.py @@ -0,0 +1,153 @@ +import click +import os +import sys +import logging +import subprocess + +from typing import Optional + +from aim.sdk.repo import Repo, RepoStatus +from aim.web.configs import AIM_ENV_MODE_KEY +from aim.web.configs import AIM_LOG_LEVEL_KEY + + +class ShellCommandException(Exception): + pass + + +def exec_cmd( + cmd, throw_on_error=True, env=None, stream_output=False, cwd=None, cmd_stdin=None, **kwargs +): + """ + Runs a command as a child process. + A convenience wrapper for running a command from a Python script. + Keyword arguments: + cmd -- the command to run, as a list of strings + throw_on_error -- if true, raises an Exception if the exit code of the program is nonzero + env -- additional environment variables to be defined when running the child process + cwd -- working directory for child process + stream_output -- if true, does not capture standard output and error; if false, captures these + streams and returns them + cmd_stdin -- if specified, passes the specified string as stdin to the child process. + Note on the return value: If stream_output is true, then only the exit code is returned. If + stream_output is false, then a tuple of the exit code, standard output and standard error is + returned. + """ + cmd_env = os.environ.copy() + if env: + cmd_env.update(env) + if stream_output: + child = subprocess.Popen( + cmd, env=cmd_env, cwd=cwd, universal_newlines=True, stdin=subprocess.PIPE, **kwargs + ) + child.communicate(cmd_stdin) + exit_code = child.wait() + if throw_on_error and exit_code != 0: + raise ShellCommandException("Non-zero exitcode: %s" % (exit_code)) + return exit_code + else: + child = subprocess.Popen( + cmd, + env=cmd_env, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cwd, + universal_newlines=True, + **kwargs + ) + (stdout, stderr) = child.communicate(cmd_stdin) + exit_code = child.wait() + if throw_on_error and exit_code != 0: + raise ShellCommandException( + "Non-zero exit code: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" % (exit_code, stdout, stderr) + ) + return exit_code, stdout, stderr + + +def set_log_level(log_level): + numeric_level = getattr(logging, log_level.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: %s' % log_level) + os.environ[AIM_LOG_LEVEL_KEY] = str(numeric_level) + logging.basicConfig(level=numeric_level) + + +def build_db_upgrade_command(): + from aim import web + web_dir = os.path.dirname(web.__file__) + migrations_dir = os.path.join(web_dir, 'migrations') + if os.getenv(AIM_ENV_MODE_KEY, 'prod') == 'prod': + ini_file = os.path.join(migrations_dir, 'alembic.ini') + else: + ini_file = os.path.join(migrations_dir, 'alembic_dev.ini') + return [sys.executable, '-m', 'alembic', '-c', ini_file, 'upgrade', 'head'] + + +def build_uvicorn_command(app, + host='0.0.0.0', + port=0, + workers=1, + uds=None, + ssl_keyfile=None, + ssl_certfile=None, + log_level='warning', + ): + cmd = [sys.executable, '-m', 'uvicorn', + '--host', host, '--port', f'{port}', + '--workers', f'{workers}'] + + if os.getenv(AIM_ENV_MODE_KEY, 'prod') == 'prod': + log_level = log_level or 'error' + else: + import aim + cmd += ['--reload', '--reload-dir', os.path.dirname(aim.__file__)] + log_level = log_level or 'debug' + + if uds: + cmd += ['--uds', uds] + if ssl_keyfile: + cmd += ['--ssl-keyfile', ssl_keyfile] + if ssl_certfile: + cmd += ['--ssl-certfile', ssl_certfile] + cmd += ['--log-level', log_level.lower()] + cmd += [app] + return cmd + + +def get_free_port_num(): + import socket + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(('', 0)) + port_num = s.getsockname()[1] + s.close() + return port_num + + +def get_repo_instance(repo_path: str, yes: bool) -> Optional['Repo']: + repo_status = Repo.check_repo_status(repo_path) + if repo_status == RepoStatus.MISSING: + if yes: + init_repo = True + else: + init_repo = click.confirm(f'\'{repo_path}\' is not a valid Aim repository. Do you want to initialize it?') + if not init_repo: + click.echo('To initialize repo please run the following command:') + click.secho('aim init', fg='yellow') + return + repo_inst = Repo.from_path(repo_path, init=True) + elif repo_status == RepoStatus.UPDATE_REQUIRED: + if yes: + reinit_repo = True + else: + reinit_repo = click.confirm('Found non-empty \'.aim\' directory. Would you like to overwrite it?') + if not reinit_repo: + click.echo('To re-initialize repo please run the following command:') + click.secho('aim init', fg='yellow') + return + Repo.rm(repo_path) + repo_inst = Repo.from_path(repo_path, init=True) + else: + repo_inst = Repo.from_path(repo_path) + + return repo_inst diff --git a/aim/cli/version/__init__.py b/aim/cli/version/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/version/commands.py b/aim/cli/version/commands.py new file mode 100644 index 0000000..0b7ba48 --- /dev/null +++ b/aim/cli/version/commands.py @@ -0,0 +1,8 @@ +import click + +from aim.__version__ import __version__ as aim_version + + +@click.command() +def version(): + click.echo('Aim v{}'.format(aim_version)) diff --git a/aim/cli/watcher_cli.py b/aim/cli/watcher_cli.py new file mode 100644 index 0000000..81503e2 --- /dev/null +++ b/aim/cli/watcher_cli.py @@ -0,0 +1,296 @@ +import logging + +import click +import uuid + +from click import core +from collections import OrderedDict +from typing import Optional, Mapping + +from aim.ext.notifier import get_config +from aim.ext.notifier.utils import has_watcher_config, set_default_config +from aim.sdk.run_status_watcher import RunStatusWatcher +from aim.sdk.repo import Repo + + +core._verify_python3_env = lambda: None +DEFAULT_MESSAGE_TEMPLATE = "❗️ Something wrong with Run '{run.hash}'. Please check. ❗️" +MESSAGE_PROMPT = "Stuck Runs notification message" + + +class OrderedGroup(click.Group): + def __init__(self, name: Optional[str] = None, commands: Optional[Mapping[str, click.Command]] = None, **kwargs): + super(OrderedGroup, self).__init__(name, commands, **kwargs) + #: the registered subcommands by their exported names. + self.commands = commands or OrderedDict() + + def list_commands(self, ctx: click.Context) -> Mapping[str, click.Command]: + return self.commands + + +def check_configuration(ctx: click.Context, repo: Repo) -> bool: + if not has_watcher_config(repo.path): + click.echo(f'Repo \'{repo.path}\' has no configured notifiers.') + if click.confirm('Would you like to configure notifiers?', default=True): + ctx.invoke(add_config) + elif click.confirm('Would you like to use default configuration?'): + set_default_config(repo.path) + else: + return False + return True + + +def dump_notifier_config(cfg: dict): + lines = ['', '', f'Type: {cfg["type"]}'] + for arg_name, value in cfg['arguments'].items(): + printable_name = arg_name.replace('_', ' ').capitalize() + lines.append(f'{printable_name}: {value}') + lines.append('--------') + click.echo('\n'.join(lines)) + + +@click.group() +@click.option('--repo', + required=False, + help='Aim Repo to check Run statuses.', + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True)) +@click.pass_context +def cli_entry_point(ctx, repo): + """Service for detecting and reporting training Run failures.""" + repo_path = repo or Repo.default_repo_path() + repo = Repo.from_path(repo_path) + + ctx.ensure_object(dict) + ctx.obj['repo'] = repo + ctx.obj['config'] = get_config(repo.path) + + +@cli_entry_point.command(name='start') +@click.pass_context +def start_watcher(ctx): + """Start watcher service to monitor and report stuck/failed Runs.""" + repo = ctx.obj['repo'] + if check_configuration(ctx, repo): + watcher = RunStatusWatcher(repo) + click.secho(f'Starting Aim watcher for repo \'{repo.path}\'...', fg='yellow') + click.echo('Press Ctrl+C to exit') + + watcher.start_watcher() + + +@cli_entry_point.group(cls=OrderedGroup, name='notifiers') +@click.pass_context +def config_notifiers(ctx): + """Configure how notifications should be received.""" + pass + + +@click.command(name='dump', hidden=True) +@click.pass_context +def dump_config(ctx): + """Dump notifier configuration file.""" + cfg = ctx.obj['config'] + if not cfg.exists(): + repo = ctx.obj['repo'] + click.echo(f'Cannot find notifier configuration for Repo \'{repo.path}\'.') + return + + click.echo(cfg.dump()) + + +@click.command(name='list') +@click.pass_context +def list_config(ctx): + """List available notifiers.""" + cfg = ctx.obj['config'] + if not cfg.exists(): + repo = ctx.obj['repo'] + click.echo(f'Cannot find notifier configuration for Repo \'{repo.path}\'.') + return + + click.echo("{:<40} {:<10} {:<10}".format('NOTIFIER ID', 'TYPE', 'STATUS')) + for notifier in cfg.notifiers.values(): + click.echo("{:<40} {:<10} {:<10}".format(notifier['id'], notifier['type'], notifier['status'])) + + +@click.command(name='get-log-level') +@click.pass_context +def get_log_level(ctx): + """Get Log Notifications level.""" + cfg = ctx.obj['config'] + if not cfg.exists(): + repo = ctx.obj['repo'] + click.echo(f'Cannot find notifier configuration for Repo \'{repo.path}\'.') + return + + click.echo(f'Log level: {logging.getLevelName(cfg.log_level)}') + + +def get_level_names(): + available_levels = (logging.CRITICAL, logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG) + return tuple(map(logging.getLevelName, available_levels)) + + +@click.command(name='set-log-level') +@click.argument('level', required=True, type=click.Choice(get_level_names())) +@click.pass_context +def set_log_level(ctx, level): + """Set Log Notifications level to .""" + cfg = ctx.obj['config'] + if not cfg.exists(): + repo = ctx.obj['repo'] + click.echo(f'Cannot find notifier configuration for Repo \'{repo.path}\'.') + return + + cfg.log_level = getattr(logging, level) + cfg.save() + + +@click.group(name='add', invoke_without_command=True) +@click.pass_context +def add_config(ctx): + """Add a new notifier configuration (slack, workplace, etc.).""" + if ctx.invoked_subcommand is None: + add_new = True + while add_new: + sub_commands = ctx.command.list_commands(ctx) + choice = click.prompt('Select notifier type to add:', show_choices=True, type=click.Choice(sub_commands)) + sub_cmd = ctx.command.get_command(ctx, choice) + extra_args = {} + for param in sub_cmd.params: + if isinstance(param, click.Option) and param.prompt: + extra_args[param.name] = param.prompt_for_value(ctx) + ctx.invoke(sub_cmd, **extra_args) + add_new = click.confirm('Would you like to add another notifier?') + + +@click.command(name='remove') +@click.argument('notifier-id', required=True, type=str) +@click.pass_context +def remove_config(ctx, notifier_id): + """Remove notifier configuration from the list.""" + cfg = ctx.obj['config'] + if not cfg.exists(): + repo = ctx.obj['repo'] + click.echo(f'Cannot find notifier configuration for Repo \'{repo.path}\'.') + return + + if notifier_id in cfg.notifiers: + dump_notifier_config(cfg.get(notifier_id)) + click.confirm('Remove notifier configuration above?', abort=True) + cfg.remove(notifier_id) + cfg.save() + else: + click.echo(f'No notifier with id {notifier_id} found.') + + +@click.command(name='enable') +@click.argument('notifier-id', required=True, type=str) +@click.pass_context +def enable_config(ctx, notifier_id): + """Start receiving notifications from given notifier.""" + cfg = ctx.obj['config'] + if not cfg.exists(): + repo = ctx.obj['repo'] + click.echo(f'Cannot find notifier configuration for Repo \'{repo.path}\'.') + return + + if notifier_id in cfg.notifiers: + cfg.enable(notifier_id) + cfg.save() + else: + click.echo(f'No notifier with id {notifier_id} found.') + + +@click.command(name='disable') +@click.argument('notifier-id', required=True, type=str) +@click.pass_context +def disable_config(ctx, notifier_id): + """Stop receiving notifications from given notifier.""" + cfg = ctx.obj['config'] + if not cfg.exists(): + repo = ctx.obj['repo'] + click.echo(f'Cannot find notifier configuration for Repo \'{repo.path}\'.') + return + + if notifier_id in cfg.notifiers: + cfg.disable(notifier_id) + cfg.save() + else: + click.echo(f'No notifier with id {notifier_id} found.') + + +@add_config.command(name='workplace') +@click.option('--group-id', prompt=True, required=True, type=int) +@click.option('--access-token', prompt=True, required=True, type=str) +@click.option('--message', prompt=MESSAGE_PROMPT, required=False, type=str, + default=DEFAULT_MESSAGE_TEMPLATE, show_default=True) +@click.pass_context +def workplace_config(ctx, group_id, access_token, message): + cfg = ctx.obj['config'] + new_cfg = { + 'id': str(uuid.uuid4()), + 'type': 'workplace', + 'arguments': { + 'group_id': group_id, + 'access_token': access_token, + 'message': message, + } + } + dump_notifier_config(new_cfg) + click.confirm('Save notifier configuration above?', default=True, abort=True) + cfg.add(new_cfg) + cfg.save() + + +@add_config.command(name='slack') +@click.option('--webhook-url', prompt=True, required=True, type=str) +@click.option('--message', prompt=MESSAGE_PROMPT, required=False, type=str, + default=DEFAULT_MESSAGE_TEMPLATE, show_default=True) +@click.pass_context +def slack_config(ctx, webhook_url, message): + cfg = ctx.obj['config'] + new_cfg = { + 'id': str(uuid.uuid4()), + 'type': 'slack', + 'arguments': { + 'url': webhook_url, + 'message': message, + } + } + dump_notifier_config(new_cfg) + click.confirm('Save notifier configuration above?', default=True, abort=True) + cfg.add(new_cfg) + cfg.save() + + +@add_config.command(name='logger') +@click.option('--message', prompt=MESSAGE_PROMPT, required=False, type=str, + default=DEFAULT_MESSAGE_TEMPLATE, show_default=True) +@click.pass_context +def logger_config(ctx, message): + cfg = ctx.obj['config'] + new_cfg = { + 'id': str(uuid.uuid4()), + 'type': 'logger', + 'arguments': { + 'message': message, + } + } + dump_notifier_config(new_cfg) + click.confirm('Save notifier configuration above?', default=True, abort=True) + cfg.add(new_cfg) + cfg.save() + + +config_notifiers.add_command(add_config) +config_notifiers.add_command(list_config) +config_notifiers.add_command(remove_config) +config_notifiers.add_command(disable_config) +config_notifiers.add_command(enable_config) +config_notifiers.add_command(dump_config) +config_notifiers.add_command(get_log_level) +config_notifiers.add_command(set_log_level) diff --git a/aim/ext/__init__.py b/aim/ext/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/ext/cleanup/__init__.py b/aim/ext/cleanup/__init__.py new file mode 100644 index 0000000..aac43b8 --- /dev/null +++ b/aim/ext/cleanup/__init__.py @@ -0,0 +1,102 @@ +import time +from abc import abstractmethod + +import atexit +import weakref +import logging +import threading + +from typing import Generic, Tuple, TypeVar, Dict + +T = TypeVar('T') + +logger = logging.getLogger(__name__) + + +class RobustExec(threading.Thread): + """ + A thread that executes a function and catches all soft interrupts. + Users very often Ctrl-C to stop the program multiple times which leaves + program no chance to clean up. + """ + def __init__(self, *args, stop_signal, **kwargs): + super().__init__(*args, **kwargs) + self.stop_signal = stop_signal + + def join(self): + while True: + try: + # Pressing Ctrl-C will raise KeyboardInterrupt and stop only + # the `Event.wait()` call. The target function of `RobustExec` + # will continue to run, until stop_signal is set. + return self.stop_signal.wait() + except KeyboardInterrupt: + logger.warning('Received Ctrl-C. Closing gracefully.') + + +class AutoClean(Generic[T]): + PRIORITY = 10 + _registered_with_atexit = False + _finalizers: Dict[ + T, Tuple[int, weakref.finalize] + ] = weakref.WeakKeyDictionary() + + stop_signal = threading.Event() + + def __init__(self, instance: T) -> None: + """ + Initialize object resources that need to be cleaned up automatically. + """ + self.finalizer = weakref.finalize(instance, self._close) + # We do not finalizer to automatically close the object itself. + # Instead, we will collect all finalizers and call them at exit. + self.finalizer.atexit = False + self._finalizers[instance] = (self.PRIORITY, self.finalizer) + + if not AutoClean._registered_with_atexit: + atexit.register(AutoClean.cleanup) + AutoClean._registered_with_atexit = True + + def close(self) -> None: + """Manually close the object.""" + self.finalizer() + + @abstractmethod + def _close(self): + """ + This method is called when the instance is garbage collected. + Meant to be overridden by subclasses. + """ + ... + + @staticmethod + def _cleanup(): + """ + Actually cleanup all the remaining resources in order of specified + priorities. Resources with bigger priority numbers are cleaned up first (have higher priority) + while the resources with equal priorities are cleaned up in reverse order + of their registration (hence why reversed is used). + This is called automatically by atexit. It is not meant tobe called manually. + """ + + finalizers = sorted(AutoClean._finalizers.items(), key=lambda x: x[1][0]) + logger.debug(f'Cleaning up... Found {len(finalizers)} finalizers') + logger.debug('Cleaning up... Iterating over instances in order') + for key, (priority, finalizer) in reversed(finalizers): + logger.debug(f'Cleaning up... with priority={priority} instance {key}') + finalizer() + AutoClean.stop_signal.set() + + @staticmethod + def cleanup(): + """ + Cleanup all the remaining objects. This is called automatically + when the program exits. + It also blocks until all the cleanup functions have finished to ensure + data consistency. + """ + logger.debug('Cleaning up... Blocking KeyboardInterrupts') + example = RobustExec(stop_signal=AutoClean.stop_signal, target=AutoClean._cleanup) + example.start() + example.join() + logger.debug('Cleaning up... Done') diff --git a/aim/ext/exception_resistant.py b/aim/ext/exception_resistant.py new file mode 100644 index 0000000..347bf53 --- /dev/null +++ b/aim/ext/exception_resistant.py @@ -0,0 +1,72 @@ +import logging + +from functools import wraps + + +logger = logging.getLogger(__name__) + + +def exception_resistant(silent: bool): + def inner(func): + if not silent: + num_fails = 0 + max_fails = 6 + + @wraps(func) + def wrapper(*args, **kwargs): + nonlocal num_fails + func_name = func.__name__ + try: + return func(*args, **kwargs) + except Exception as e: + num_fails += 1 + if num_fails == 1: + print('Something went wrong in `{}`. The process will continue to execute.'.format(func_name)) + if num_fails <= max_fails: + print('`{}`: {}'.format(func_name, e)) + elif num_fails == max_fails + 1: + print('The rest of the `{}` errors are hidden.'.format(func_name)) + else: + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception: + pass + return wrapper + return inner + + +class _SafeModeConfig: + @staticmethod + def log_exception(e: Exception, func: callable): + logger.warning(f'Exception "{str(e)}" raised in function "{func.__name__}"') + + @staticmethod + def reraise_exception(e: Exception, func: callable): + raise e + + exception_callback = reraise_exception + + +def enable_safe_mode(): + _SafeModeConfig.exception_callback = _SafeModeConfig.log_exception + + +def disable_safe_mode(): + _SafeModeConfig.exception_callback = _SafeModeConfig.reraise_exception + + +def set_exception_callback(callback: callable): + _SafeModeConfig.exception_callback = callback + + +def noexcept(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + _SafeModeConfig.exception_callback(e, func) + + return wrapper diff --git a/aim/ext/notebook/__init__.py b/aim/ext/notebook/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/ext/notebook/notebook.py b/aim/ext/notebook/notebook.py new file mode 100644 index 0000000..4a19ed5 --- /dev/null +++ b/aim/ext/notebook/notebook.py @@ -0,0 +1,223 @@ +import shlex + +from aim.cli.configs import VERSION_NAME, UP_NAME +import aim.cli.manager.manager as manager + +# Error message prefix for aim commands +ERROR_MSG_PREFIX = b'Error:' + +# returned by get_execution_context +_COLAB_EXEC_CONTEXT = "_COLAB_EXEC_CONTEXT" +_IPYTHON_EXEC_CONTEXT = "_IPYTHON_EXEC_CONTEXT" +_OTHER_EXEC_CONTEXT = "_OTHER_EXEC_CONTEXT" + +# current execution context +_CURRENT_CONTEXT = _OTHER_EXEC_CONTEXT + +# environment specific constants +# useful for detecting the environment from the UI +_SAGE_MAKER_NOTEBOOK_PATH_POSTFIX = "/aim-sage" +_NOTEBOOK_PATH_POSTFIX = "/notebook" + + +def get_execution_context(): + """Determine the most specific context that we're in. + Returns: + _COLAB_EXEC_CONTEXT: If in Colab with an IPython notebook context. + _IPYTHON_EXEC_CONTEXT: If we are in an IPython notebook + context but not in colab (i.e. `jupyter notebook`) + line). + _OTHER_EXEC_CONTEXT: Otherwise (e.g., by running a Python script at the + command-line or using the `ipython` interactive shell). + """ + # In Colab, the `google.colab` module is available, but the shell + # returned by `IPython.get_ipython` does not have a `get_trait` + # method. + + # imports are dynamic, since some modules are not available for all contexts + try: + import IPython + except ImportError: + pass + else: + ipython = IPython.get_ipython() + # @TODO find a stable way to get colab context + if ipython is not None and 'google.colab' in str(ipython): + # We are in Colab notebook context + # global _CURRENT_CONTEXT + # _CURRENT_CONTEXT = _COLAB_EXEC_CONTEXT + return _COLAB_EXEC_CONTEXT + + # In an IPython command line shell or Jupyter notebook + elif ipython is not None and ipython.has_trait("kernel"): + # global _CURRENT_CONTEXT + # _CURRENT_CONTEXT = _IPYTHON_EXEC_CONTEXT + return _IPYTHON_EXEC_CONTEXT + + # Otherwise, we're not in a known notebook context. + return _OTHER_EXEC_CONTEXT + + +def get_argument_options(line): + """ + Returns parsed argument options and command from magic cell as dict (command, options) + currently parse only --=value style to dict + Set default values for the required fields, otherwise the provided fields + Will omit unsupported args @TODO notify about unsupported args + @TODO add process args all styles to dict + """ + # @TODO improve this logic + # --proxy-url is useful to print the right url, and set UI's url into iframe correctly + supported_args = ['--port', '--host', '--repo', '--proxy-url'] + + args = shlex.split(line) + command = args[0] + + options = { + '--host': '127.0.0.1', + '--port': '43801', + '--base-path': _NOTEBOOK_PATH_POSTFIX + } + for arg in args[1:]: + key, value = arg.split('=', 1) + if key in supported_args: + options[key] = value + # if --proxy-url passed + if options.get('--proxy-url'): + options['--base-path'] = f'/proxy/absolute/{options["--port"]}{_SAGE_MAKER_NOTEBOOK_PATH_POSTFIX}' + + return command, options + + +def display_colab(port, display): + """Display Aim instance in a Colab output frame. + It need go through the proxy + """ + import IPython.display + + shell = """ + (async () => {{ + const url = new URL('{path}/', await google.colab.kernel.proxyPort({port}, {{'cache': true}})); + const iframe = document.createElement('iframe'); + iframe.src = url; + const a = document.createElement('a'); + a.href = url; + a.innerHTML = 'Open in new browser tab'; + a.setAttribute('target', '_blank'); + + iframe.setAttribute('width', '100%'); + iframe.setAttribute('height', '800'); + iframe.setAttribute('frameborder', 0); + document.body.appendChild(iframe); + document.body.appendChild(a); + }})(); + """.format(path=_NOTEBOOK_PATH_POSTFIX, port=port) + + script = IPython.display.Javascript(shell) + + if display: + display.update(script) + else: + IPython.display.display(script) + + +def display_notebook(host, port, display, proxy_url=None): + """Display Aim instance in an ipython context output frame. + """ + import IPython.display + url = "{}:{}{}".format(host, port, _NOTEBOOK_PATH_POSTFIX) + + # @TODO add warning if proxy_url is not defined + if proxy_url: + # jupyter-server-proxy supports absolute paths by using it with /proxy/absolute/ path + url = "{}{}{}{}/".format(proxy_url, '/proxy/absolute/', port, _SAGE_MAKER_NOTEBOOK_PATH_POSTFIX) + print(url) + + shell = """ + + """.format(url) + + iframe = IPython.display.HTML(shell) + display.update(iframe) + + +def up(options, context): + """ + Calls to run `aim up` command width corresponding arguments + Handles the result of the command + Renders the