From aadb1071fc9bca1a83bf8bbeb92bcd1924aeda97 Mon Sep 17 00:00:00 2001 From: unknown <2569982966@qq.com> Date: Sat, 7 Oct 2023 14:54:02 +0800 Subject: [PATCH] test --- .github/FUNDING.yml | 1 + .github/ISSUE_TEMPLATE.md | 63 + .github/workflows/core.yml | 52 + .github/workflows/deployment.yml | 45 + .github/workflows/docs.yml | 109 ++ .github/workflows/examples.yml | 63 + .github/workflows/projects.yml | 69 + .gitignore | 12 + .gitmodules | 7 + .pylintrc | 12 + CODE_OF_CONDUCT.md | 3 + CONTRIBUTING.md | 22 + HISTORY.rst | 324 +++++ LICENSE | 174 +++ Makefile | 38 + README.rst | 103 ++ SECURITY.md | 34 + platformio/__init__.py | 73 + platformio/__main__.py | 144 ++ platformio/account/__init__.py | 13 + platformio/account/cli.py | 44 + platformio/account/client.py | 352 +++++ platformio/account/commands/__init__.py | 13 + platformio/account/commands/destroy.py | 37 + platformio/account/commands/forgot.py | 29 + platformio/account/commands/login.py | 26 + platformio/account/commands/logout.py | 24 + platformio/account/commands/password.py | 26 + platformio/account/commands/register.py | 52 + platformio/account/commands/show.py | 116 ++ platformio/account/commands/token.py | 32 + platformio/account/commands/update.py | 59 + platformio/account/org/__init__.py | 13 + platformio/account/org/cli.py | 38 + platformio/account/org/commands/__init__.py | 13 + platformio/account/org/commands/add.py | 34 + platformio/account/org/commands/create.py | 38 + platformio/account/org/commands/destroy.py | 34 + platformio/account/org/commands/list.py | 48 + platformio/account/org/commands/remove.py | 34 + platformio/account/org/commands/update.py | 50 + platformio/account/team/__init__.py | 13 + platformio/account/team/cli.py | 38 + platformio/account/team/commands/__init__.py | 13 + platformio/account/team/commands/add.py | 38 + platformio/account/team/commands/create.py | 37 + platformio/account/team/commands/destroy.py | 40 + platformio/account/team/commands/list.py | 59 + platformio/account/team/commands/remove.py | 36 + platformio/account/team/commands/update.py | 51 + platformio/account/validate.py | 84 ++ platformio/app.py | 293 ++++ platformio/assets/schema/library.json | 487 +++++++ .../assets/system/99-platformio-udev.rules | 173 +++ platformio/builder/__init__.py | 13 + platformio/builder/main.py | 250 ++++ platformio/builder/tools/__init__.py | 13 + platformio/builder/tools/pioasm.py | 29 + platformio/builder/tools/piobuild.py | 390 ++++++ platformio/builder/tools/piohooks.py | 50 + platformio/builder/tools/pioino.py | 256 ++++ platformio/builder/tools/piointegration.py | 193 +++ platformio/builder/tools/piolib.py | 1209 ++++++++++++++++ platformio/builder/tools/piomaxlen.py | 96 ++ platformio/builder/tools/piomisc.py | 149 ++ platformio/builder/tools/pioplatform.py | 248 ++++ platformio/builder/tools/pioproject.py | 54 + platformio/builder/tools/piosize.py | 266 ++++ platformio/builder/tools/piotarget.py | 116 ++ platformio/builder/tools/piotest.py | 61 + platformio/builder/tools/pioupload.py | 258 ++++ platformio/cache.py | 165 +++ platformio/check/__init__.py | 13 + platformio/check/cli.py | 334 +++++ platformio/check/defect.py | 95 ++ platformio/check/tools/__init__.py | 33 + platformio/check/tools/base.py | 267 ++++ platformio/check/tools/clangtidy.py | 90 ++ platformio/check/tools/cppcheck.py | 271 ++++ platformio/check/tools/pvsstudio.py | 251 ++++ platformio/cli.py | 110 ++ platformio/commands/__init__.py | 13 + platformio/commands/boards.py | 86 ++ platformio/commands/ci.py | 174 +++ platformio/commands/device/__init__.py | 18 + platformio/commands/lib.py | 459 +++++++ platformio/commands/platform.py | 391 ++++++ platformio/commands/settings.py | 80 ++ platformio/commands/update.py | 38 + platformio/commands/upgrade.py | 153 +++ platformio/compat.py | 134 ++ platformio/debug/__init__.py | 13 + platformio/debug/cli.py | 180 +++ platformio/debug/config/__init__.py | 13 + platformio/debug/config/base.py | 247 ++++ platformio/debug/config/blackmagic.py | 71 + platformio/debug/config/factory.py | 48 + platformio/debug/config/generic.py | 38 + platformio/debug/config/jlink.py | 46 + platformio/debug/config/mspdebug.py | 36 + platformio/debug/config/native.py | 33 + platformio/debug/config/qemu.py | 37 + platformio/debug/config/renode.py | 45 + platformio/debug/exception.py | 36 + platformio/debug/helpers.py | 160 +++ platformio/debug/process/__init__.py | 13 + platformio/debug/process/base.py | 154 +++ platformio/debug/process/client.py | 104 ++ platformio/debug/process/gdb.py | 181 +++ platformio/debug/process/server.py | 147 ++ platformio/device/__init__.py | 13 + platformio/device/cli.py | 30 + platformio/device/finder.py | 257 ++++ platformio/device/list/__init__.py | 13 + platformio/device/list/command.py | 99 ++ platformio/device/list/util.py | 165 +++ platformio/device/monitor/__init__.py | 13 + platformio/device/monitor/command.py | 177 +++ platformio/device/monitor/filters/__init__.py | 13 + platformio/device/monitor/filters/base.py | 100 ++ platformio/device/monitor/filters/hexlify.py | 38 + platformio/device/monitor/filters/log2file.py | 47 + .../device/monitor/filters/send_on_enter.py | 38 + platformio/device/monitor/filters/time.py | 37 + platformio/device/monitor/terminal.py | 188 +++ platformio/exception.py | 110 ++ platformio/fs.py | 226 +++ platformio/home/__init__.py | 13 + platformio/home/cli.py | 118 ++ platformio/home/rpc/__init__.py | 13 + platformio/home/rpc/handlers/__init__.py | 13 + platformio/home/rpc/handlers/account.py | 30 + platformio/home/rpc/handlers/app.py | 86 ++ platformio/home/rpc/handlers/base.py | 17 + platformio/home/rpc/handlers/ide.py | 86 ++ platformio/home/rpc/handlers/misc.py | 50 + platformio/home/rpc/handlers/os.py | 172 +++ platformio/home/rpc/handlers/piocore.py | 229 ++++ platformio/home/rpc/handlers/platform.py | 140 ++ platformio/home/rpc/handlers/project.py | 386 ++++++ platformio/home/rpc/handlers/registry.py | 31 + platformio/home/rpc/server.py | 131 ++ platformio/home/run.py | 103 ++ platformio/http.py | 221 +++ platformio/maintenance.py | 282 ++++ platformio/package/__init__.py | 13 + platformio/package/cli.py | 48 + platformio/package/commands/__init__.py | 13 + platformio/package/commands/exec.py | 109 ++ platformio/package/commands/install.py | 336 +++++ platformio/package/commands/list.py | 229 ++++ platformio/package/commands/outdated.py | 216 +++ platformio/package/commands/pack.py | 45 + platformio/package/commands/publish.py | 206 +++ platformio/package/commands/search.py | 77 ++ platformio/package/commands/show.py | 154 +++ platformio/package/commands/uninstall.py | 237 ++++ platformio/package/commands/unpublish.py | 46 + platformio/package/commands/update.py | 250 ++++ platformio/package/download.py | 165 +++ platformio/package/exception.py | 68 + platformio/package/lockfile.py | 117 ++ platformio/package/manager/__init__.py | 13 + platformio/package/manager/_download.py | 100 ++ platformio/package/manager/_install.py | 293 ++++ platformio/package/manager/_legacy.py | 61 + platformio/package/manager/_registry.py | 197 +++ platformio/package/manager/_symlink.py | 71 + platformio/package/manager/_uninstall.py | 97 ++ platformio/package/manager/_update.py | 137 ++ platformio/package/manager/base.py | 341 +++++ platformio/package/manager/core.py | 93 ++ platformio/package/manager/library.py | 124 ++ platformio/package/manager/platform.py | 180 +++ platformio/package/manager/tool.py | 30 + platformio/package/manifest/__init__.py | 13 + platformio/package/manifest/parser.py | 715 ++++++++++ platformio/package/manifest/schema.py | 284 ++++ platformio/package/meta.py | 528 +++++++ platformio/package/pack.py | 258 ++++ platformio/package/unpack.py | 209 +++ platformio/package/vcsclient.py | 273 ++++ platformio/package/version.py | 59 + platformio/platform/__init__.py | 13 + platformio/platform/_packages.py | 92 ++ platformio/platform/_run.py | 183 +++ platformio/platform/base.py | 239 ++++ platformio/platform/board.py | 145 ++ platformio/platform/exception.py | 46 + platformio/platform/factory.py | 103 ++ platformio/proc.py | 219 +++ platformio/project/__init__.py | 13 + platformio/project/cli.py | 32 + platformio/project/commands/__init__.py | 13 + platformio/project/commands/config.py | 104 ++ platformio/project/commands/init.py | 408 ++++++ platformio/project/commands/metadata.py | 81 ++ platformio/project/config.py | 524 +++++++ platformio/project/exception.py | 54 + platformio/project/helpers.py | 196 +++ platformio/project/integration/__init__.py | 13 + platformio/project/integration/generator.py | 176 +++ .../integration/tpls/clion/.gitignore.tpl | 1 + .../tpls/codeblocks/platformio.cbp.tpl | 64 + .../integration/tpls/eclipse/.cproject.tpl | 316 +++++ .../integration/tpls/eclipse/.project.tpl | 27 + .../.settings/PlatformIO Debugger.launch.tpl | 38 + .../.settings/language.settings.xml.tpl | 36 + .../.settings/org.eclipse.cdt.core.prefs.tpl | 11 + .../project/integration/tpls/emacs/.ccls.tpl | 14 + .../integration/tpls/emacs/.gitignore.tpl | 3 + .../netbeans/nbproject/configurations.xml.tpl | 73 + .../nbproject/private/configurations.xml.tpl | 61 + .../nbproject/private/launcher.properties.tpl | 40 + .../nbproject/private/private.xml.tpl | 10 + .../tpls/netbeans/nbproject/project.xml.tpl | 26 + .../integration/tpls/qtcreator/.gitignore.tpl | 2 + .../integration/tpls/qtcreator/Makefile.tpl | 12 + .../tpls/qtcreator/platformio.cflags.tpl | 3 + .../tpls/qtcreator/platformio.config.tpl | 8 + .../tpls/qtcreator/platformio.creator.tpl | 2 + .../tpls/qtcreator/platformio.cxxflags.tpl | 3 + .../tpls/qtcreator/platformio.files.tpl | 6 + .../tpls/qtcreator/platformio.includes.tpl | 4 + .../integration/tpls/sublimetext/.ccls.tpl | 14 + .../platformio.sublime-project.tpl | 108 ++ .../project/integration/tpls/vim/.ccls.tpl | 14 + .../integration/tpls/vim/.gitignore.tpl | 4 + .../platformio.vcxproj.filters.tpl | 29 + .../tpls/visualstudio/platformio.vcxproj.tpl | 75 + .../integration/tpls/vscode/.gitignore.tpl | 5 + .../vscode/.vscode/c_cpp_properties.json.tpl | 138 ++ .../tpls/vscode/.vscode/extensions.json.tpl | 35 + .../tpls/vscode/.vscode/launch.json.tpl | 99 ++ platformio/project/options.py | 852 ++++++++++++ platformio/project/savedeps.py | 87 ++ platformio/public.py | 29 + platformio/registry/__init__.py | 13 + platformio/registry/access/__init__.py | 13 + platformio/registry/access/cli.py | 36 + .../registry/access/commands/__init__.py | 13 + platformio/registry/access/commands/grant.py | 39 + platformio/registry/access/commands/list.py | 58 + .../registry/access/commands/private.py | 33 + platformio/registry/access/commands/public.py | 33 + platformio/registry/access/commands/revoke.py | 38 + platformio/registry/access/validate.py | 34 + platformio/registry/client.py | 162 +++ platformio/registry/mirror.py | 98 ++ platformio/remote/__init__.py | 13 + platformio/remote/ac/__init__.py | 13 + platformio/remote/ac/base.py | 90 ++ platformio/remote/ac/process.py | 42 + platformio/remote/ac/psync.py | 66 + platformio/remote/ac/serial.py | 60 + platformio/remote/cli.py | 412 ++++++ platformio/remote/client/__init__.py | 13 + platformio/remote/client/agent_list.py | 38 + platformio/remote/client/agent_service.py | 228 ++++ platformio/remote/client/async_base.py | 65 + platformio/remote/client/base.py | 192 +++ platformio/remote/client/device_list.py | 54 + platformio/remote/client/device_monitor.py | 239 ++++ platformio/remote/client/run_or_test.py | 271 ++++ platformio/remote/client/update_core.py | 22 + platformio/remote/factory/__init__.py | 13 + platformio/remote/factory/client.py | 86 ++ platformio/remote/factory/ssl.py | 41 + platformio/remote/projectsync.py | 117 ++ platformio/run/__init__.py | 13 + platformio/run/cli.py | 336 +++++ platformio/run/helpers.py | 45 + platformio/run/processor.py | 91 ++ platformio/system/__init__.py | 13 + platformio/system/cli.py | 32 + platformio/system/commands/__init__.py | 13 + platformio/system/commands/completion.py | 69 + platformio/system/commands/info.py | 84 ++ platformio/system/commands/prune.py | 70 + platformio/system/completion.py | 101 ++ platformio/system/prune.py | 98 ++ platformio/telemetry.py | 380 ++++++ platformio/test/__init__.py | 13 + platformio/test/cli.py | 203 +++ platformio/test/exception.py | 33 + platformio/test/helpers.py | 73 + platformio/test/reports/__init__.py | 13 + platformio/test/reports/base.py | 34 + platformio/test/reports/json.py | 100 ++ platformio/test/reports/junit.py | 107 ++ platformio/test/reports/stdout.py | 99 ++ platformio/test/result.py | 166 +++ platformio/test/runners/__init__.py | 13 + platformio/test/runners/base.py | 215 +++ platformio/test/runners/doctest.py | 121 ++ platformio/test/runners/factory.py | 67 + platformio/test/runners/googletest.py | 109 ++ platformio/test/runners/readers/__init__.py | 13 + platformio/test/runners/readers/native.py | 136 ++ platformio/test/runners/readers/serial.py | 81 ++ platformio/test/runners/unity.py | 303 ++++ platformio/util.py | 208 +++ scripts/docspregen.py | 1216 +++++++++++++++++ scripts/fixsymlink.py | 40 + scripts/install_devplatforms.py | 62 + setup.py | 96 ++ tests/__init__.py | 13 + tests/commands/__init__.py | 13 + tests/commands/pkg/__init__.py | 13 + tests/commands/pkg/test_exec.py | 59 + tests/commands/pkg/test_install.py | 592 ++++++++ tests/commands/pkg/test_list.py | 137 ++ tests/commands/pkg/test_outdated.py | 62 + tests/commands/pkg/test_search.py | 58 + tests/commands/pkg/test_show.py | 103 ++ tests/commands/pkg/test_uninstall.py | 408 ++++++ tests/commands/pkg/test_update.py | 374 +++++ tests/commands/test_account_org_team.py | 524 +++++++ tests/commands/test_boards.py | 52 + tests/commands/test_check.py | 759 ++++++++++ tests/commands/test_ci.py | 193 +++ tests/commands/test_init.py | 217 +++ tests/commands/test_lib.py | 234 ++++ tests/commands/test_lib_complex.py | 344 +++++ tests/commands/test_platform.py | 141 ++ tests/commands/test_run.py | 446 ++++++ tests/commands/test_settings.py | 24 + tests/commands/test_test.py | 660 +++++++++ tests/conftest.py | 150 ++ tests/misc/__init__.py | 13 + tests/misc/ino2cpp/__init__.py | 13 + tests/misc/ino2cpp/examples/basic/basic.ino | 60 + .../misc/ino2cpp/examples/multifiles/bar.ino | 4 + .../misc/ino2cpp/examples/multifiles/foo.pde | 13 + .../ino2cpp/examples/strmultilines/main.ino | 79 ++ tests/misc/ino2cpp/test_ino2cpp.py | 48 + tests/misc/test_maintenance.py | 50 + tests/misc/test_misc.py | 48 + tests/package/__init__.py | 13 + tests/package/test_manager.py | 654 +++++++++ tests/package/test_manifest.py | 905 ++++++++++++ tests/package/test_meta.py | 337 +++++ tests/package/test_pack.py | 188 +++ tests/project/__init__.py | 13 + tests/project/test_config.py | 717 ++++++++++ tests/project/test_savedeps.py | 224 +++ tests/test_examples.py | 102 ++ tox.ini | 71 + 348 files changed, 43895 insertions(+) create mode 100644 .github/FUNDING.yml create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/workflows/core.yml create mode 100644 .github/workflows/deployment.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/examples.yml create mode 100644 .github/workflows/projects.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .pylintrc create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 HISTORY.rst create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.rst create mode 100644 SECURITY.md create mode 100644 platformio/__init__.py create mode 100644 platformio/__main__.py create mode 100644 platformio/account/__init__.py create mode 100644 platformio/account/cli.py create mode 100644 platformio/account/client.py create mode 100644 platformio/account/commands/__init__.py create mode 100644 platformio/account/commands/destroy.py create mode 100644 platformio/account/commands/forgot.py create mode 100644 platformio/account/commands/login.py create mode 100644 platformio/account/commands/logout.py create mode 100644 platformio/account/commands/password.py create mode 100644 platformio/account/commands/register.py create mode 100644 platformio/account/commands/show.py create mode 100644 platformio/account/commands/token.py create mode 100644 platformio/account/commands/update.py create mode 100644 platformio/account/org/__init__.py create mode 100644 platformio/account/org/cli.py create mode 100644 platformio/account/org/commands/__init__.py create mode 100644 platformio/account/org/commands/add.py create mode 100644 platformio/account/org/commands/create.py create mode 100644 platformio/account/org/commands/destroy.py create mode 100644 platformio/account/org/commands/list.py create mode 100644 platformio/account/org/commands/remove.py create mode 100644 platformio/account/org/commands/update.py create mode 100644 platformio/account/team/__init__.py create mode 100644 platformio/account/team/cli.py create mode 100644 platformio/account/team/commands/__init__.py create mode 100644 platformio/account/team/commands/add.py create mode 100644 platformio/account/team/commands/create.py create mode 100644 platformio/account/team/commands/destroy.py create mode 100644 platformio/account/team/commands/list.py create mode 100644 platformio/account/team/commands/remove.py create mode 100644 platformio/account/team/commands/update.py create mode 100644 platformio/account/validate.py create mode 100644 platformio/app.py create mode 100644 platformio/assets/schema/library.json create mode 100644 platformio/assets/system/99-platformio-udev.rules create mode 100644 platformio/builder/__init__.py create mode 100644 platformio/builder/main.py create mode 100644 platformio/builder/tools/__init__.py create mode 100644 platformio/builder/tools/pioasm.py create mode 100644 platformio/builder/tools/piobuild.py create mode 100644 platformio/builder/tools/piohooks.py create mode 100644 platformio/builder/tools/pioino.py create mode 100644 platformio/builder/tools/piointegration.py create mode 100644 platformio/builder/tools/piolib.py create mode 100644 platformio/builder/tools/piomaxlen.py create mode 100644 platformio/builder/tools/piomisc.py create mode 100644 platformio/builder/tools/pioplatform.py create mode 100644 platformio/builder/tools/pioproject.py create mode 100644 platformio/builder/tools/piosize.py create mode 100644 platformio/builder/tools/piotarget.py create mode 100644 platformio/builder/tools/piotest.py create mode 100644 platformio/builder/tools/pioupload.py create mode 100644 platformio/cache.py create mode 100644 platformio/check/__init__.py create mode 100644 platformio/check/cli.py create mode 100644 platformio/check/defect.py create mode 100644 platformio/check/tools/__init__.py create mode 100644 platformio/check/tools/base.py create mode 100644 platformio/check/tools/clangtidy.py create mode 100644 platformio/check/tools/cppcheck.py create mode 100644 platformio/check/tools/pvsstudio.py create mode 100644 platformio/cli.py create mode 100644 platformio/commands/__init__.py create mode 100644 platformio/commands/boards.py create mode 100644 platformio/commands/ci.py create mode 100644 platformio/commands/device/__init__.py create mode 100644 platformio/commands/lib.py create mode 100644 platformio/commands/platform.py create mode 100644 platformio/commands/settings.py create mode 100644 platformio/commands/update.py create mode 100644 platformio/commands/upgrade.py create mode 100644 platformio/compat.py create mode 100644 platformio/debug/__init__.py create mode 100644 platformio/debug/cli.py create mode 100644 platformio/debug/config/__init__.py create mode 100644 platformio/debug/config/base.py create mode 100644 platformio/debug/config/blackmagic.py create mode 100644 platformio/debug/config/factory.py create mode 100644 platformio/debug/config/generic.py create mode 100644 platformio/debug/config/jlink.py create mode 100644 platformio/debug/config/mspdebug.py create mode 100644 platformio/debug/config/native.py create mode 100644 platformio/debug/config/qemu.py create mode 100644 platformio/debug/config/renode.py create mode 100644 platformio/debug/exception.py create mode 100644 platformio/debug/helpers.py create mode 100644 platformio/debug/process/__init__.py create mode 100644 platformio/debug/process/base.py create mode 100644 platformio/debug/process/client.py create mode 100644 platformio/debug/process/gdb.py create mode 100644 platformio/debug/process/server.py create mode 100644 platformio/device/__init__.py create mode 100644 platformio/device/cli.py create mode 100644 platformio/device/finder.py create mode 100644 platformio/device/list/__init__.py create mode 100644 platformio/device/list/command.py create mode 100644 platformio/device/list/util.py create mode 100644 platformio/device/monitor/__init__.py create mode 100644 platformio/device/monitor/command.py create mode 100644 platformio/device/monitor/filters/__init__.py create mode 100644 platformio/device/monitor/filters/base.py create mode 100644 platformio/device/monitor/filters/hexlify.py create mode 100644 platformio/device/monitor/filters/log2file.py create mode 100644 platformio/device/monitor/filters/send_on_enter.py create mode 100644 platformio/device/monitor/filters/time.py create mode 100644 platformio/device/monitor/terminal.py create mode 100644 platformio/exception.py create mode 100644 platformio/fs.py create mode 100644 platformio/home/__init__.py create mode 100644 platformio/home/cli.py create mode 100644 platformio/home/rpc/__init__.py create mode 100644 platformio/home/rpc/handlers/__init__.py create mode 100644 platformio/home/rpc/handlers/account.py create mode 100644 platformio/home/rpc/handlers/app.py create mode 100644 platformio/home/rpc/handlers/base.py create mode 100644 platformio/home/rpc/handlers/ide.py create mode 100644 platformio/home/rpc/handlers/misc.py create mode 100644 platformio/home/rpc/handlers/os.py create mode 100644 platformio/home/rpc/handlers/piocore.py create mode 100644 platformio/home/rpc/handlers/platform.py create mode 100644 platformio/home/rpc/handlers/project.py create mode 100644 platformio/home/rpc/handlers/registry.py create mode 100644 platformio/home/rpc/server.py create mode 100644 platformio/home/run.py create mode 100644 platformio/http.py create mode 100644 platformio/maintenance.py create mode 100644 platformio/package/__init__.py create mode 100644 platformio/package/cli.py create mode 100644 platformio/package/commands/__init__.py create mode 100644 platformio/package/commands/exec.py create mode 100644 platformio/package/commands/install.py create mode 100644 platformio/package/commands/list.py create mode 100644 platformio/package/commands/outdated.py create mode 100644 platformio/package/commands/pack.py create mode 100644 platformio/package/commands/publish.py create mode 100644 platformio/package/commands/search.py create mode 100644 platformio/package/commands/show.py create mode 100644 platformio/package/commands/uninstall.py create mode 100644 platformio/package/commands/unpublish.py create mode 100644 platformio/package/commands/update.py create mode 100644 platformio/package/download.py create mode 100644 platformio/package/exception.py create mode 100644 platformio/package/lockfile.py create mode 100644 platformio/package/manager/__init__.py create mode 100644 platformio/package/manager/_download.py create mode 100644 platformio/package/manager/_install.py create mode 100644 platformio/package/manager/_legacy.py create mode 100644 platformio/package/manager/_registry.py create mode 100644 platformio/package/manager/_symlink.py create mode 100644 platformio/package/manager/_uninstall.py create mode 100644 platformio/package/manager/_update.py create mode 100644 platformio/package/manager/base.py create mode 100644 platformio/package/manager/core.py create mode 100644 platformio/package/manager/library.py create mode 100644 platformio/package/manager/platform.py create mode 100644 platformio/package/manager/tool.py create mode 100644 platformio/package/manifest/__init__.py create mode 100644 platformio/package/manifest/parser.py create mode 100644 platformio/package/manifest/schema.py create mode 100644 platformio/package/meta.py create mode 100644 platformio/package/pack.py create mode 100644 platformio/package/unpack.py create mode 100644 platformio/package/vcsclient.py create mode 100644 platformio/package/version.py create mode 100644 platformio/platform/__init__.py create mode 100644 platformio/platform/_packages.py create mode 100644 platformio/platform/_run.py create mode 100644 platformio/platform/base.py create mode 100644 platformio/platform/board.py create mode 100644 platformio/platform/exception.py create mode 100644 platformio/platform/factory.py create mode 100644 platformio/proc.py create mode 100644 platformio/project/__init__.py create mode 100644 platformio/project/cli.py create mode 100644 platformio/project/commands/__init__.py create mode 100644 platformio/project/commands/config.py create mode 100644 platformio/project/commands/init.py create mode 100644 platformio/project/commands/metadata.py create mode 100644 platformio/project/config.py create mode 100644 platformio/project/exception.py create mode 100644 platformio/project/helpers.py create mode 100644 platformio/project/integration/__init__.py create mode 100644 platformio/project/integration/generator.py create mode 100644 platformio/project/integration/tpls/clion/.gitignore.tpl create mode 100644 platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl create mode 100644 platformio/project/integration/tpls/eclipse/.cproject.tpl create mode 100644 platformio/project/integration/tpls/eclipse/.project.tpl create mode 100644 platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl create mode 100644 platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl create mode 100644 platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl create mode 100644 platformio/project/integration/tpls/emacs/.ccls.tpl create mode 100644 platformio/project/integration/tpls/emacs/.gitignore.tpl create mode 100644 platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl create mode 100644 platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl create mode 100644 platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl create mode 100644 platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl create mode 100644 platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl create mode 100644 platformio/project/integration/tpls/qtcreator/.gitignore.tpl create mode 100644 platformio/project/integration/tpls/qtcreator/Makefile.tpl create mode 100644 platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl create mode 100644 platformio/project/integration/tpls/qtcreator/platformio.config.tpl create mode 100644 platformio/project/integration/tpls/qtcreator/platformio.creator.tpl create mode 100644 platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl create mode 100644 platformio/project/integration/tpls/qtcreator/platformio.files.tpl create mode 100644 platformio/project/integration/tpls/qtcreator/platformio.includes.tpl create mode 100644 platformio/project/integration/tpls/sublimetext/.ccls.tpl create mode 100644 platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl create mode 100644 platformio/project/integration/tpls/vim/.ccls.tpl create mode 100644 platformio/project/integration/tpls/vim/.gitignore.tpl create mode 100644 platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl create mode 100644 platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl create mode 100644 platformio/project/integration/tpls/vscode/.gitignore.tpl create mode 100644 platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl create mode 100644 platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl create mode 100644 platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl create mode 100644 platformio/project/options.py create mode 100644 platformio/project/savedeps.py create mode 100644 platformio/public.py create mode 100644 platformio/registry/__init__.py create mode 100644 platformio/registry/access/__init__.py create mode 100644 platformio/registry/access/cli.py create mode 100644 platformio/registry/access/commands/__init__.py create mode 100644 platformio/registry/access/commands/grant.py create mode 100644 platformio/registry/access/commands/list.py create mode 100644 platformio/registry/access/commands/private.py create mode 100644 platformio/registry/access/commands/public.py create mode 100644 platformio/registry/access/commands/revoke.py create mode 100644 platformio/registry/access/validate.py create mode 100644 platformio/registry/client.py create mode 100644 platformio/registry/mirror.py create mode 100644 platformio/remote/__init__.py create mode 100644 platformio/remote/ac/__init__.py create mode 100644 platformio/remote/ac/base.py create mode 100644 platformio/remote/ac/process.py create mode 100644 platformio/remote/ac/psync.py create mode 100644 platformio/remote/ac/serial.py create mode 100644 platformio/remote/cli.py create mode 100644 platformio/remote/client/__init__.py create mode 100644 platformio/remote/client/agent_list.py create mode 100644 platformio/remote/client/agent_service.py create mode 100644 platformio/remote/client/async_base.py create mode 100644 platformio/remote/client/base.py create mode 100644 platformio/remote/client/device_list.py create mode 100644 platformio/remote/client/device_monitor.py create mode 100644 platformio/remote/client/run_or_test.py create mode 100644 platformio/remote/client/update_core.py create mode 100644 platformio/remote/factory/__init__.py create mode 100644 platformio/remote/factory/client.py create mode 100644 platformio/remote/factory/ssl.py create mode 100644 platformio/remote/projectsync.py create mode 100644 platformio/run/__init__.py create mode 100644 platformio/run/cli.py create mode 100644 platformio/run/helpers.py create mode 100644 platformio/run/processor.py create mode 100644 platformio/system/__init__.py create mode 100644 platformio/system/cli.py create mode 100644 platformio/system/commands/__init__.py create mode 100644 platformio/system/commands/completion.py create mode 100644 platformio/system/commands/info.py create mode 100644 platformio/system/commands/prune.py create mode 100644 platformio/system/completion.py create mode 100644 platformio/system/prune.py create mode 100644 platformio/telemetry.py create mode 100644 platformio/test/__init__.py create mode 100644 platformio/test/cli.py create mode 100644 platformio/test/exception.py create mode 100644 platformio/test/helpers.py create mode 100644 platformio/test/reports/__init__.py create mode 100644 platformio/test/reports/base.py create mode 100644 platformio/test/reports/json.py create mode 100644 platformio/test/reports/junit.py create mode 100644 platformio/test/reports/stdout.py create mode 100644 platformio/test/result.py create mode 100644 platformio/test/runners/__init__.py create mode 100644 platformio/test/runners/base.py create mode 100644 platformio/test/runners/doctest.py create mode 100644 platformio/test/runners/factory.py create mode 100644 platformio/test/runners/googletest.py create mode 100644 platformio/test/runners/readers/__init__.py create mode 100644 platformio/test/runners/readers/native.py create mode 100644 platformio/test/runners/readers/serial.py create mode 100644 platformio/test/runners/unity.py create mode 100644 platformio/util.py create mode 100644 scripts/docspregen.py create mode 100644 scripts/fixsymlink.py create mode 100644 scripts/install_devplatforms.py create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/commands/__init__.py create mode 100644 tests/commands/pkg/__init__.py create mode 100644 tests/commands/pkg/test_exec.py create mode 100644 tests/commands/pkg/test_install.py create mode 100644 tests/commands/pkg/test_list.py create mode 100644 tests/commands/pkg/test_outdated.py create mode 100644 tests/commands/pkg/test_search.py create mode 100644 tests/commands/pkg/test_show.py create mode 100644 tests/commands/pkg/test_uninstall.py create mode 100644 tests/commands/pkg/test_update.py create mode 100644 tests/commands/test_account_org_team.py create mode 100644 tests/commands/test_boards.py create mode 100644 tests/commands/test_check.py create mode 100644 tests/commands/test_ci.py create mode 100644 tests/commands/test_init.py create mode 100644 tests/commands/test_lib.py create mode 100644 tests/commands/test_lib_complex.py create mode 100644 tests/commands/test_platform.py create mode 100644 tests/commands/test_run.py create mode 100644 tests/commands/test_settings.py create mode 100644 tests/commands/test_test.py create mode 100644 tests/conftest.py create mode 100644 tests/misc/__init__.py create mode 100644 tests/misc/ino2cpp/__init__.py create mode 100644 tests/misc/ino2cpp/examples/basic/basic.ino create mode 100644 tests/misc/ino2cpp/examples/multifiles/bar.ino create mode 100644 tests/misc/ino2cpp/examples/multifiles/foo.pde create mode 100644 tests/misc/ino2cpp/examples/strmultilines/main.ino create mode 100644 tests/misc/ino2cpp/test_ino2cpp.py create mode 100644 tests/misc/test_maintenance.py create mode 100644 tests/misc/test_misc.py create mode 100644 tests/package/__init__.py create mode 100644 tests/package/test_manager.py create mode 100644 tests/package/test_manifest.py create mode 100644 tests/package/test_meta.py create mode 100644 tests/package/test_pack.py create mode 100644 tests/project/__init__.py create mode 100644 tests/project/test_config.py create mode 100644 tests/project/test_savedeps.py create mode 100644 tests/test_examples.py create mode 100644 tox.ini diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..6f70f7e --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://platformio.org/donate diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..d490fe6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,63 @@ +What kind of issue is this? + +- [ ] **Question**. + This issue tracker is not the place for questions. If you want to ask how to do something, + or to understand why something isn't working the way you expect it to, + use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support) + +- [ ] **PlatformIO IDE**. + All issues related to PlatformIO IDE should be reported to the + [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) repository + +- [ ] **Development Platform or Board**. + All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms + should be reported to appropriate repository related to your hardware + https://github.com/topics/platformio-platform + +- [ ] **Feature Request**. + Start by telling us what problem you’re trying to solve. Often a solution + already exists! Don’t send pull requests to implement new features without first getting our + support. Sometimes we leave features out on purpose to keep the project small. + +- [ ] **PlatformIO Core**. + If you’ve found a bug, please provide an information below. + + +*You can erase any parts of this template not applicable to your Issue.* + +------------------------------------------------------------------ + +### Configuration + +**Operating system**: + +**PlatformIO Version** (`platformio --version`): + +### Description of problem + + +#### Steps to Reproduce + +1. +2. +3. + +### Actual Results + + +### Expected Results + + +### If problems with PlatformIO Build System: + +**The content of `platformio.ini`:** +```ini +Insert here... +``` + +**Source file to reproduce issue:** +```cpp +Insert here... +``` + +### Additional info diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml new file mode 100644 index 0000000..c7c020a --- /dev/null +++ b/.github/workflows/core.yml @@ -0,0 +1,52 @@ +name: Core + +on: [push, pull_request] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, windows-latest, macos-latest] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox + + - name: Core System Info + run: | + tox -e py + + - name: Python Lint + if: ${{ matrix.python-version != '3.6' }} + run: | + tox -e lint + + - name: Integration Tests + if: ${{ matrix.python-version == '3.9' }} + run: | + tox -e testcore + + - name: Slack Notification + uses: homoluctus/slatify@master + if: failure() + with: + type: ${{ job.status }} + job_name: '*Core*' + commit: true + url: ${{ secrets.SLACK_BUILD_WEBHOOK }} + token: ${{ secrets.SLACK_GITHUB_TOKEN }} diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml new file mode 100644 index 0000000..370333a --- /dev/null +++ b/.github/workflows/deployment.yml @@ -0,0 +1,45 @@ +name: Deployment + +on: + push: + branches: + - "master" + - "release/**" + +jobs: + deployment: + runs-on: ubuntu-latest + environment: production + + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox wheel + + - name: Deployment Tests + env: + TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }} + TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }} + TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }} + run: | + tox -e testcore + + - name: Build Python source tarball + run: python setup.py sdist bdist_wheel + + - name: Publish package to PyPI + if: ${{ github.ref == 'refs/heads/master' }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..9a64d88 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,109 @@ +name: Docs + +on: [push, pull_request] + +jobs: + build: + name: Build Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox + + - name: Build docs + run: | + tox -e docs + + - name: Slack Notification + uses: homoluctus/slatify@master + if: failure() + with: + type: ${{ job.status }} + job_name: '*Docs*' + commit: true + url: ${{ secrets.SLACK_BUILD_WEBHOOK }} + token: ${{ secrets.SLACK_GITHUB_TOKEN }} + + - name: Preserve Docs + if: ${{ github.event_name == 'push' }} + run: | + tar -czvf docs.tar.gz -C docs/_build html rtdpage + + - name: Save artifact + if: ${{ github.event_name == 'push' }} + uses: actions/upload-artifact@v3 + with: + name: docs + path: ./docs.tar.gz + + deploy: + name: Deploy Docs + needs: build + runs-on: ubuntu-latest + env: + DOCS_REPO: platformio/platformio-docs + DOCS_DIR: platformio-docs + LATEST_DOCS_DIR: latest-docs + RELEASE_BUILD: ${{ startsWith(github.ref, 'refs/tags/v') }} + if: ${{ github.event_name == 'push' }} + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: docs + - name: Unpack artifact + run: | + mkdir ./${{ env.LATEST_DOCS_DIR }} + tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }} + - name: Delete Artifact + uses: geekyeggo/delete-artifact@v2 + with: + name: docs + - name: Select Docs type + id: get-destination-dir + run: | + if [[ ${{ env.RELEASE_BUILD }} == true ]]; then + echo "::set-output name=dst_dir::stable" + else + echo "::set-output name=dst_dir::latest" + fi + - name: Checkout latest Docs + continue-on-error: true + uses: actions/checkout@v3 + with: + repository: ${{ env.DOCS_REPO }} + path: ${{ env.DOCS_DIR }} + ref: gh-pages + - name: Synchronize Docs + run: | + rm -rf ${{ env.DOCS_DIR }}/.git + rm -rf ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} + mkdir -p ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} + cp -rf ${{ env.LATEST_DOCS_DIR }}/html/* ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }} + if [[ ${{ env.RELEASE_BUILD }} == false ]]; then + rm -rf ${{ env.DOCS_DIR }}/page + mkdir -p ${{ env.DOCS_DIR }}/page + cp -rf ${{ env.LATEST_DOCS_DIR }}/rtdpage/* ${{ env.DOCS_DIR }}/page + fi + - name: Validate Docs + run: | + if [ -z "$(ls -A ${{ env.DOCS_DIR }})" ]; then + echo "Docs folder is empty. Aborting!" + exit 1 + fi + - name: Deploy to Github Pages + uses: peaceiris/actions-gh-pages@v3 + with: + personal_token: ${{ secrets.DEPLOY_GH_DOCS_TOKEN }} + external_repository: ${{ env.DOCS_REPO }} + publish_dir: ./${{ env.DOCS_DIR }} + commit_message: Sync Docs diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml new file mode 100644 index 0000000..6bb258e --- /dev/null +++ b/.github/workflows/examples.yml @@ -0,0 +1,63 @@ +name: Examples + +on: [push, pull_request] + + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + env: + PIO_INSTALL_DEVPLATFORM_OWNERNAMES: "platformio" + PIO_INSTALL_DEVPLATFORM_NAMES: "aceinna_imu,atmelavr,atmelmegaavr,atmelsam,espressif32,espressif8266,nordicnrf52,raspberrypi,ststm32,teensy" + + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox + + - name: Run on Linux + if: startsWith(matrix.os, 'ubuntu') + run: | + # Free space + sudo apt clean + docker rmi $(docker image ls -aq) + df -h + tox -e testexamples + + - name: Run on macOS + if: startsWith(matrix.os, 'macos') + run: | + df -h + tox -e testexamples + + - name: Run on Windows + if: startsWith(matrix.os, 'windows') + env: + PLATFORMIO_CORE_DIR: C:/pio + PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH + run: | + tox -e testexamples + + - name: Slack Notification + uses: homoluctus/slatify@master + if: failure() + with: + type: ${{ job.status }} + job_name: '*Examples*' + commit: true + url: ${{ secrets.SLACK_BUILD_WEBHOOK }} + token: ${{ secrets.SLACK_GITHUB_TOKEN }} diff --git a/.github/workflows/projects.yml b/.github/workflows/projects.yml new file mode 100644 index 0000000..1d37c5a --- /dev/null +++ b/.github/workflows/projects.yml @@ -0,0 +1,69 @@ +name: Projects + +on: [push, pull_request] + +jobs: + build: + strategy: + fail-fast: false + matrix: + project: + - marlin: + repository: "MarlinFirmware/Marlin" + folder: "Marlin" + config_dir: "Marlin" + env_name: "mega2560" + - esphome: + repository: "esphome/esphome" + folder: "esphome" + config_dir: "esphome" + env_name: "esp32-arduino" + - smartknob: + repository: "scottbez1/smartknob" + folder: "smartknob" + config_dir: "smartknob" + env_name: "view" + - espurna: + repository: "xoseperez/espurna" + folder: "espurna" + config_dir: "espurna/code" + env_name: "nodemcu-lolin" + - OpenMQTTGateway: + repository: "1technophile/OpenMQTTGateway" + folder: "OpenMQTTGateway" + config_dir: "OpenMQTTGateway" + env_name: "esp32-m5atom-lite" + os: [ubuntu-latest, windows-latest, macos-latest] + exclude: + - os: windows-latest + project: {"esphome": "", "repository": "esphome/esphome", "folder": "esphome", "config_dir": "esphome", "env_name": "esp32-arduino"} + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install PlatformIO + run: pip install -U . + + - name: Check out ${{ matrix.project.repository }} + uses: actions/checkout@v3 + with: + submodules: "recursive" + repository: ${{ matrix.project.repository }} + path: ${{ matrix.project.folder }} + + - name: Install ESPHome dependencies + # Requires esptool package as it's used in a custom prescript + if: ${{ contains(matrix.project.repository, 'esphome') }} + run: pip install esptool==3.* + + - name: Compile ${{ matrix.project.repository }} + run: pio run -d ${{ matrix.project.config_dir }} -e ${{ matrix.project.env_name }} + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ceeaed9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +*.egg-info +*.pyc +__pycache__ +.tox +docs/_build +dist +build +.cache +coverage.xml +.coverage +htmlcov +.pytest_cache diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..6dac16a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,7 @@ +[submodule "examples"] + path = examples + url = https://github.com/platformio/platformio-examples.git +[submodule "docs"] + path = docs + url = https://github.com/platformio/platformio-docs.git + branch = develop diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..172b43a --- /dev/null +++ b/.pylintrc @@ -0,0 +1,12 @@ +[REPORTS] +output-format=colorized + +[MESSAGES CONTROL] +disable= + missing-docstring, + duplicate-code, + invalid-name, + too-few-public-methods, + consider-using-f-string, + cyclic-import, + use-dict-literal diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..1958f6c --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +See https://piolabs.com/legal/code-of-conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8deafb9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,22 @@ +Contributing +------------ + +To get started, sign the Contributor License Agreement. + +1. Fork the repository on GitHub +2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git` +3. Run `pip install tox` +4. Go to the root of the PlatformIO Core project where `tox.ini` is located (``cd platformio-core``) and run `tox -e py39`. + You can replace `py39` with your own Python version. For example, `py311` means Python 3.11. +5. Activate current development environment: + + * Windows: `.tox\py39\Scripts\activate` + * Bash/ZSH: `source .tox/py39/bin/activate` + * Fish: `source .tox/py39/bin/activate.fish` + +6. Make changes to code, documentation, etc. +7. Lint source code `make before-commit` +8. Run the tests `make test` +9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html) +10. Commit changes to your forked repository +11. Submit a Pull Request on GitHub diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..6a5b2a0 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,324 @@ +Release Notes +============= + +.. |PIOCONF| replace:: `"platformio.ini" `__ configuration file +.. |LIBRARYJSON| replace:: `library.json `__ +.. |LDF| replace:: `LDF `__ +.. |INTERPOLATION| replace:: `Interpolation of Values `__ +.. |UNITTESTING| replace:: `Unit Testing `__ +.. |DEBUGGING| replace:: `Debugging `__ + +.. _release_notes_6: + +PlatformIO Core 6 +----------------- + +Unlock the true potential of embedded software development with +PlatformIO's collaborative ecosystem, embracing declarative principles, +test-driven methodologies, and modern toolchains for unrivaled success. + +6.1.11 (2023-08-31) +~~~~~~~~~~~~~~~~~~~ + +* Resolved a possible issue that may cause generated projects for `PlatformIO IDE for VSCode `__ to fail to launch a debug session because of a missing "objdump" binary when GDB is not part of the toolchain package +* Resolved a regression issue that resulted in the malfunction of the Memory Inspection feature within `PIO Home `__ + +6.1.10 (2023-08-11) +~~~~~~~~~~~~~~~~~~~ + +* Resolved an issue that caused generated projects for `PlatformIO IDE for VSCode `__ to break when the ``-iprefix`` compiler flag was used +* Resolved an issue encountered while utilizing the `pio pkg exec `__ command on the Windows platform to execute Python scripts from a package +* Implemented a crucial improvement to the `pio run `__ command, guaranteeing that the ``monitor`` target is not executed if any of the preceding targets, such as ``upload``, encounter failures +* `Cppcheck `__ v2.11 with new checks, CLI commands and various analysis improvements +* Resolved a critical issue that arose on macOS ARM platforms due to the Python "requests" module, leading to a "ModuleNotFoundError: No module named 'chardet'" (`issue #4702 `_) + +6.1.9 (2023-07-06) +~~~~~~~~~~~~~~~~~~ + +* Rectified a regression bug that occurred when the ``-include`` flag was passed via the `build_flags `__ option as a relative path and subsequently expanded (`issue #4683 `_) +* Resolved an issue that resulted in unresolved absolute toolchain paths when generating the `Compilation database "compile_commands.json" `__ (`issue #4684 `_) + +6.1.8 (2023-07-05) +~~~~~~~~~~~~~~~~~~ + +* Added a new ``--lint`` option to the `pio project config `__ command, enabling users to efficiently perform linting on the |PIOCONF| +* Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information +* Expanded the functionality of the |LIBRARYJSON| manifest by allowing the use of the underscore symbol in the `keywords `__ field +* Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 `_) +* Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 `_) +* Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 `_) +* Updated the `pio project metadata `__ command to return C/C++ flags as parsed Unix shell arguments when dumping project build metadata +* Resolved a critical issue related to the usage of the ``-include`` flag within the `build_flags `__ option, specifically when employing dynamic variables (`issue #4682 `_) +* Removed PlatformIO IDE for Atom from the documentation as `Atom has been deprecated `__ + +6.1.7 (2023-05-08) +~~~~~~~~~~~~~~~~~~ + +* Introduced a new ``--sample-code`` option to the `pio project init `__ command, which allows users to include sample code in the newly created project +* Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) +* Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) +* Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process +* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) +* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 `_) +* Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to +* Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) +* Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) +* Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow +* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 `_, `issue #4493 `_, `issue #4607 `_) +* Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) +* Resolved an issue where organization details could not be updated using the `pio org update `__ command +* Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) +* Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) +* Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 `_) +* Resolved an issue where multiple targets were not executed sequentially (`issue #4604 `_) +* Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 (`issue #4540 `_) + +6.1.6 (2023-01-23) +~~~~~~~~~~~~~~~~~~ + +* Added support for Python 3.11 +* Added a new `name `__ configuration option to customize a project name (`pull #4498 `_) +* Made assets (templates, ``99-platformio-udev.rules``) part of Python's module (`issue #4458 `_) +* Updated `Clang-Tidy `__ check tool to v15.0.5 with new diagnostics and bugfixes +* Removed dependency on the "zeroconf" package and install it only when a user lists mDNS devices (issue with zeroconf's LGPL license) +* Show the real error message instead of "Can not remove temporary directory" when |PIOCONF| is broken (`issue #4480 `_) +* Fixed an issue with an incorrect test summary when a testcase name includes a colon (`issue #4508 `_) +* Fixed an issue when `extends `__ did not override options in the right order (`issue #4462 `_) +* Fixed an issue when `pio pkg list `__ and `pio pkg uninstall `__ commands fail if there are circular dependencies in the |LIBRARYJSON| manifests (`issue #4475 `_) + +6.1.5 (2022-11-01) +~~~~~~~~~~~~~~~~~~ + +* Added a new `enable_proxy_strict_ssl `__ setting to disable the proxy server certificate verification (`issue #4432 `_) +* Documented `PlatformIO Core Proxy Configuration `__ +* Speeded up device port finder by avoiding loading board HWIDs from development platforms +* Improved caching of build metadata in debug mode +* Fixed an issue when `pio pkg install --storage-dir `__ command requires PlatformIO project (`issue #4410 `_) + +6.1.4 (2022-08-12) +~~~~~~~~~~~~~~~~~~ + +* Added support for accepting the original FileNode environment in a "callback" function when using `Build Middlewares `__ (`pull #4380 `_) +* Improved device port finder when using dual channel UART converter (`issue #4367 `_) +* Improved project dependency resolving when using the `pio project init --ide `__ command +* Upgraded build engine to the SCons 4.4.0 (`release notes `__) +* Keep custom "unwantedRecommendations" when generating projects for VSCode (`issue #4383 `_) +* Do not resolve project dependencies for the ``cleanall`` target (`issue #4344 `_) +* Warn about calling "env.BuildSources" in a POST-type script (`issue #4385 `_) +* Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 `_) +* Fixed an issue when the "cleanall" target removes dependencies from all working environments (`issue #4386 `_) + +6.1.3 (2022-07-18) +~~~~~~~~~~~~~~~~~~ + +* Fixed a regression bug when opening device monitor without any filters (`issue #4363 `_) + +6.1.2 (2022-07-18) +~~~~~~~~~~~~~~~~~~ + +* Export a ``PIO_UNIT_TESTING`` macro to the project source files and dependent libraries in the |UNITTESTING| mode +* Improved detection of Windows architecture (`issue #4353 `_) +* Warn about unknown `device monitor filters `__ (`issue #4362 `_) +* Fixed a regression bug when `libArchive `__ option declared in the |LIBRARYJSON| manifest was ignored (`issue #4351 `_) +* Fixed an issue when the `pio pkg publish `__ command didn't work with Python 3.6 (`issue #4352 `_) + +6.1.1 (2022-07-11) +~~~~~~~~~~~~~~~~~~ + +* Added new ``monitor_encoding`` project configuration option to configure `Device Monitor `__ (`issue #4350 `_) +* Allowed specifying project environments for `pio ci `__ command (`issue #4347 `_) +* Show "TimeoutError" only in the verbose mode when can not find a serial port +* Fixed an issue when a serial port was not automatically detected if the board has predefined HWIDs +* Fixed an issue with endless scanning of project dependencies (`issue #4349 `_) +* Fixed an issue with |LDF| when incompatible libraries were used for the working project environment with the missed framework (`pull #4346 `_) + +6.1.0 (2022-07-06) +~~~~~~~~~~~~~~~~~~ + +* **Device Manager** + + - Automatically reconnect device monitor if a connection fails + - Added new `pio device monitor --no-reconnect `__ option to disable automatic reconnection + - Handle device monitor disconnects more gracefully (`issue #3939 `_) + - Improved a serial port finder for `Black Magic Probe `__ (`issue #4023 `_) + - Improved a serial port finder for a board with predefined HWIDs + - Replaced ``monitor_flags`` with independent project configuration options: `monitor_parity `__, `monitor_eol `__, `monitor_raw `__, `monitor_echo `__ + - Fixed an issue when the monitor filters were not applied in their order (`issue #4320 `_) + +* **Unit Testing** + + - Updated "Getting Started" documentation for `GoogleTest `__ testing and mocking framework + - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder) + - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 `_) + - Do not resolve dependencies from the project "src" folder when the `test_build_src `__ option is not enabled + - Do not immediately terminate a testing program when results are received + - Fixed an issue when a custom `pio test --project-config `__ was not handled properly (`issue #4299 `_) + - Fixed an issue when testing results were wrong in the verbose mode (`issue #4336 `_) + +* **Build System** + + - Significantly improved support for `Pre & Post Actions `__ + + * Allowed to declare actions in the `PRE-type scripts `__ even if the target is not ready yet + * Allowed library maintainers to use Pre & Post Actions in the library `extraScript `__ + + - Documented `Stringification `__ – converting a macro argument into a string constant (`issue #4310 `_) + - Added new `pio run --monitor-port `__ option to specify custom device monitor port to the ``monitor`` target (`issue #4337 `_) + - Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting `__ + - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 `_) + - Fixed an issue when the `build_unflags `__ operation ignores a flag value (`issue #4309 `_) + - Fixed an issue when the `build_unflags `__ option was not applied to the ``ASPPFLAGS`` scope + - Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" `__ + - Fixed an issue with the |LDF| when recursively scanning dependencies in the ``chain`` mode + - Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 `_) + +* **Package Management** + + - Fixed an issue when library dependencies were installed for the incompatible project environment (`issue #4338 `_) + +* **Miscellaneous** + + - Warn about incompatible Bash version for the `Shell Completion `__ (`issue #4326 `_) + +6.0.2 (2022-06-01) +~~~~~~~~~~~~~~~~~~ + +* Control |UNITTESTING| verbosity with a new multilevel `pio test -v `__ command option (`issue #4276 `_) +* Follow symbolic links during searching for the unit test suites (`issue #4288 `_) +* Show a warning when testing an empty project without a test suite (`issue #4278 `_) +* Improved support for `Asking for input (prompts) `_ +* Fixed an issue when the `build_src_flags `__ option was applied outside the project scope (`issue #4277 `_) +* Fixed an issue with debugging assembly files without preprocessor (".s") + +6.0.1 (2022-05-17) +~~~~~~~~~~~~~~~~~~ + +* Improved support for the renamed configuration options (`issue #4270 `_) +* Fixed an issue when calling the built-in `pio device monitor `__ filters +* Fixed an issue when using |INTERPOLATION| and merging str+int options (`issue #4271 `_) + +6.0.0 (2022-05-16) +~~~~~~~~~~~~~~~~~~ + +Please check the `Migration guide from 5.x to 6.0 `__. + +* **Package Management** + + - New unified Package Management CLI (``pio pkg``): + + * `pio pkg exec `_ - run command from package tool (`issue #4163 `_) + * `pio pkg install `_ - install the project dependencies or custom packages + * `pio pkg list `__ - list installed packages + * `pio pkg outdated `__ - check for project outdated packages + * `pio pkg search `__ - search for packages + * `pio pkg show `__ - show package information + * `pio pkg uninstall `_ - uninstall the project dependencies or custom packages + * `pio pkg update `__ - update the project dependencies or custom packages + + - Package Manifest + + * Added support for `"scripts" `__ (`issue #485 `_) + * Added support for `multi-licensed `__ packages using SPDX Expressions (`issue #4037 `_) + * Added support for `"dependencies" `__ declared in a "tool" package manifest + + - Added support for `symbolic links `__ allowing pointing the local source folder to the Package Manager (`issue #3348 `_) + - Automatically install dependencies of the local (private) project libraries (`issue #2910 `_) + - Improved detection of a package type from the tarball archive (`issue #3828 `_) + - Ignore files according to the patterns declared in ".gitignore" when using the `pio package pack `__ command (`issue #4188 `_) + - Dropped automatic updates of global libraries and development platforms (`issue #4179 `_) + - Dropped support for the "pythonPackages" field in "platform.json" manifest in favor of `Extra Python Dependencies `__ + - Fixed an issue when manually removed dependencies from the |PIOCONF| were not uninstalled from the storage (`issue #3076 `_) + +* **Unit Testing** + + - Refactored from scratch |UNITTESTING| solution and its documentation + - New: `Test Hierarchy `_ (`issue #4135 `_) + - New: `Doctest `__ testing framework (`issue #4240 `_) + - New: `GoogleTest `__ testing and mocking framework (`issue #3572 `_) + - New: `Semihosting `__ (`issue #3516 `_) + - New: Hardware `Simulators `__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions) + - New: ``test`` `build configuration `__ + - Added support for a `custom testing framework `_ + - Added support for a custom `testing command `__ + - Added support for a `custom Unity library `__ (`issue #3980 `_) + - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port `__ option (`issue #4229 `_) + - List available project tests with a new `pio test --list-tests `__ option + - Pass extra arguments to the testing program with a new `pio test --program-arg `__ option (`issue #3132 `_) + - Generate reports in JUnit and JSON formats using the `pio test `__ command (`issue #2891 `_) + - Provide more information when the native program crashed on a host (errored with a non-zero return code) (`issue #3429 `_) + - Improved automatic detection of a testing serial port (`issue #4076 `_) + - Fixed an issue when command line parameters (``--ignore``, ``--filter``) do not override values defined in the |PIOCONF| (`issue #3845 `_) + - Renamed the "test_build_project_src" project configuration option to the `test_build_src `__ + - Removed the "test_transport" option in favor of the `Custom "unity_config.h" `_ + +* **Static Code Analysis** + + - Updated analysis tools: + + * `Cppcheck `__ v2.7 with various checker improvements and fixed false positives + * `PVS-Studio `__ v7.18 with improved and updated semantic analysis system + + - Added support for the custom `Clang-Tidy `__ configuration file (`issue #4186 `_) + - Added ability to override a tool version using the `platform_packages `__ option (`issue #3798 `_) + - Fixed an issue with improper handling of defects that don't specify a source file (`issue #4237 `_) + +* **Build System** + + - Show project dependency licenses when building in the verbose mode + - Fixed an issue when |LDF| ignores the project `lib_deps `__ while resolving library dependencies (`issue #3598 `_) + - Fixed an issue with calling an extra script located outside a project (`issue #4220 `_) + - Fixed an issue when GCC preprocessor was applied to the ".s" assembly files on case-sensitive OS such as Window OS (`issue #3917 `_) + - Fixed an issue when |LDF| ignores `build_src_flags `__ in the "deep+" mode (`issue #4253 `_) + +* **Integration** + + - Added a new build variable (``COMPILATIONDB_INCLUDE_TOOLCHAIN``) to include toolchain paths in the compilation database (`issue #3735 `_) + - Changed a default path for compilation database `compile_commands.json `__ to the project root + - Enhanced integration for Qt Creator (`issue #3046 `_) + +* **Project Configuration** + + - Extended |INTERPOLATION| with ``${this}`` pattern (`issue #3953 `_) + - Embed environment name of the current section in the |PIOCONF| using ``${this.__env__}`` pattern + - Renamed the "src_build_flags" project configuration option to the `build_src_flags `__ + - Renamed the "src_filter" project configuration option to the `build_src_filter `__ + +* **Miscellaneous** + + - Pass extra arguments to the `native `__ program with a new `pio run --program-arg `__ option (`issue #4246 `_) + - Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 `_) + - Finally removed all tracks to the Python 2.7, the Python 3.6 is the minimum supported version. + +.. _release_notes_5: + +PlatformIO Core 5 +----------------- + +See `PlatformIO Core 5.0 history `__. + +.. _release_notes_4: + +PlatformIO Core 4 +----------------- + +See `PlatformIO Core 4.0 history `__. + +PlatformIO Core 3 +----------------- + +See `PlatformIO Core 3.0 history `__. + +PlatformIO Core 2 +----------------- + +See `PlatformIO Core 2.0 history `__. + +PlatformIO Core 1 +----------------- + +See `PlatformIO Core 1.0 history `__. + +PlatformIO Core Preview +----------------------- + +See `PlatformIO Core Preview history `__. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2f79a3b --- /dev/null +++ b/LICENSE @@ -0,0 +1,174 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..09c0b83 --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +lint: + pylint --rcfile=./.pylintrc ./tests + pylint --rcfile=./.pylintrc ./platformio + +isort: + isort ./platformio + isort ./tests + +format: + black ./platformio + black ./tests + +test: + py.test --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py + +before-commit: isort format lint + +clean-docs: + rm -rf docs/_build + +clean: clean-docs + find . -name \*.pyc -delete + find . -name __pycache__ -delete + rm -rf .cache + rm -rf build + rm -rf htmlcov + rm -f .coverage + +profile: + # Usage $ > make PIOARGS="boards" profile + python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS} + snakeviz .tox/.tmp/cprofile.prof + +pack: + python setup.py sdist + +publish: + python setup.py sdist upload diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..df73525 --- /dev/null +++ b/README.rst @@ -0,0 +1,103 @@ +PlatformIO Core +=============== + +.. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg + :target: https://docs.platformio.org/en/latest/core/index.html + :alt: CI Build for PlatformIO Core +.. image:: https://github.com/platformio/platformio-core/workflows/Docs/badge.svg + :target: https://docs.platformio.org?utm_source=github&utm_medium=core + :alt: CI Build for Docs +.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg + :target: https://github.com/platformio/platformio-examples + :alt: CI Build for dev-platform examples +.. image:: https://github.com/platformio/platformio-core/workflows/Projects/badge.svg + :target: https://docs.platformio.org/en/latest/tutorials/index.html#projects + :alt: CI Build for the Community Projects +.. image:: https://img.shields.io/pypi/v/platformio.svg + :target: https://pypi.python.org/pypi/platformio/ + :alt: Latest Version +.. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg + :alt: PlatformIO Labs + :target: https://piolabs.com/?utm_source=github&utm_medium=core + +**Quick Links:** `Homepage `_ | +`PlatformIO IDE `_ | +`Registry `_ | +`Project Examples `__ | +`Docs `_ | +`Donate `_ | +`Contact Us `_ + +**Social:** `LinkedIn `_ | +`Twitter `_ | +`Facebook `_ | +`Community Forums `_ + +.. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png + :target: https://platformio.org?utm_source=github&utm_medium=core + +`PlatformIO `_: Your Gateway to Embedded Software Development Excellence. + +Unlock the true potential of embedded software development with +PlatformIO's collaborative ecosystem, embracing declarative principles, +test-driven methodologies, and modern toolchains for unrivaled success. + +* Open source, maximum permissive Apache 2.0 license +* Cross-platform IDE and Unified Debugger +* Static Code Analyzer and Remote Unit Testing +* Multi-platform and Multi-architecture Build System +* Firmware File Explorer and Memory Inspection + +Get Started +----------- + +* `What is PlatformIO? `_ +* `PlatformIO IDE `_ +* `PlatformIO Core (CLI) `_ +* `Project Examples `__ + +Solutions +--------- + +* `Library Management `_ +* `Desktop IDEs Integration `_ +* `Continuous Integration `_ + +**Advanced** + +* `Debugging `_ +* `Unit Testing `_ +* `Static Code Analysis `_ +* `Remote Development `_ + +Registry +-------- + +* `Libraries `_ +* `Development Platforms `_ +* `Development Tools `_ + +Contributing +------------ + +See `contributing guidelines `_. + +Telemetry / Privacy Policy +-------------------------- + +Share minimal diagnostics and usage information to help us make PlatformIO better. +It is enabled by default. For more information see: + +* `Telemetry Setting `_ + +License +------- + +Copyright (c) 2014-present PlatformIO + +The PlatformIO is licensed under the permissive Apache 2.0 license, +so you can use it in both commercial and personal projects with confidence. + +.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg + :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md + :alt: SWUbanner \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3d71c1e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,34 @@ +# Security Policy + +## Supported Versions + +We are committed to ensuring the security and protection of PlatformIO Core. +To this end, we support only the following versions: + +| Version | Supported | +| ------- | ------------------ | +| 6.1.x | :white_check_mark: | +| < 6.1 | :x: | + +Unsupported versions of the PlatformIO Core may have known vulnerabilities or security issues that could compromise the security of our organization's systems and data. +Therefore, it is important that all developers use only supported versions of the PlatformIO Core. + +## Reporting a Vulnerability + +We take the security of our systems and data very seriously. We encourage responsible disclosure of any vulnerabilities or security issues that you may find in our systems or applications. If you believe you have discovered a vulnerability, please report it to us immediately. + +To report a vulnerability, please send an email to our security team at contact@piolabs.com. Please include as much information as possible, including: + +- A description of the vulnerability and how it can be exploited +- Steps to reproduce the vulnerability +- Any additional information that can help us understand and reproduce the vulnerability + +Once we receive your report, our security team will acknowledge receipt within 24 hours and will work to validate the reported vulnerability. We will provide periodic updates on the progress of the vulnerability assessment, and will notify you once a fix has been deployed. + +If the vulnerability is accepted, we will work to remediate the issue as quickly as possible. We may also provide credit or recognition to the individual who reported the vulnerability, at our discretion. + +If the vulnerability is declined, we will provide a justification for our decision and may offer guidance on how to improve the report or how to test the system more effectively. + +Please note that we will not take any legal action against individuals who report vulnerabilities in good faith and in accordance with this policy. + +Thank you for helping us keep our systems and data secure. diff --git a/platformio/__init__.py b/platformio/__init__.py new file mode 100644 index 0000000..c0226cf --- /dev/null +++ b/platformio/__init__.py @@ -0,0 +1,73 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +VERSION = (6, 1, 11) +__version__ = ".".join([str(s) for s in VERSION]) + +__title__ = "platformio" +__description__ = ( + "Your Gateway to Embedded Software Development Excellence. " + "Unlock the true potential of embedded software development " + "with PlatformIO's collaborative ecosystem, embracing " + "declarative principles, test-driven methodologies, and " + "modern toolchains for unrivaled success." +) +__url__ = "https://platformio.org" + +__author__ = "PlatformIO Labs" +__email__ = "contact@piolabs.com" + +__license__ = "Apache Software License" +__copyright__ = "Copyright 2014-present PlatformIO Labs" + +__accounts_api__ = "https://api.accounts.platformio.org" +__registry_mirror_hosts__ = [ + "registry.platformio.org", + "registry.nm1.platformio.org", +] +__pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" + +__core_packages__ = { + "contrib-piohome": "~3.4.2", + "contrib-pioremote": "~1.0.0", + "tool-scons": "~4.40502.0", + "tool-cppcheck": "~1.21100.0", + "tool-clangtidy": "~1.150005.0", + "tool-pvs-studio": "~7.18.0", +} + +__check_internet_hosts__ = [ + "185.199.110.153", # Github.com + "88.198.170.159", # platformio.org + "github.com", +] + __registry_mirror_hosts__ + +__install_requires__ = [ + # Core requirements + "bottle == 0.12.*", + "click >=8.0.4, <=8.2", + "colorama", + "marshmallow == 3.*", + "pyelftools == 0.29", + "pyserial == 3.5.*", # keep in sync "device/monitor/terminal.py" + "requests == 2.*", + "semantic_version == 2.10.*", + "tabulate == 0.*", +] + [ + # PIO Home requirements + "ajsonrpc == 1.2.*", + "starlette >=0.19, <0.32", + "uvicorn >=0.16, <0.24", + "wsproto == 1.*", +] diff --git a/platformio/__main__.py b/platformio/__main__.py new file mode 100644 index 0000000..58cabe8 --- /dev/null +++ b/platformio/__main__.py @@ -0,0 +1,144 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import traceback + +import click + +from platformio import __version__, exception, maintenance +from platformio.cli import PlatformioCLI +from platformio.compat import IS_CYGWIN, ensure_python3 + + +@click.command( + cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"]) +) +@click.version_option(__version__, prog_name="PlatformIO Core") +@click.option("--force", "-f", is_flag=True, help="DEPRECATED", hidden=True) +@click.option("--caller", "-c", help="Caller ID (service)") +@click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters") +@click.pass_context +def cli(ctx, force, caller, no_ansi): # pylint: disable=unused-argument + try: + if ( + no_ansi + or str( + os.getenv("PLATFORMIO_NO_ANSI", os.getenv("PLATFORMIO_DISABLE_COLOR")) + ).lower() + == "true" + ): + # pylint: disable=protected-access + click._compat.isatty = lambda stream: False + elif ( + str( + os.getenv("PLATFORMIO_FORCE_ANSI", os.getenv("PLATFORMIO_FORCE_COLOR")) + ).lower() + == "true" + ): + # pylint: disable=protected-access + click._compat.isatty = lambda stream: True + except: # pylint: disable=bare-except + pass + + maintenance.on_cmd_start(ctx, caller) + + +@cli.result_callback() +@click.pass_context +def process_result(*_, **__): + maintenance.on_cmd_end() + + +def configure(): + if IS_CYGWIN: + raise exception.CygwinEnvDetected() + + # https://urllib3.readthedocs.org + # /en/latest/security.html#insecureplatformwarning + try: + import urllib3 # pylint: disable=import-outside-toplevel + + urllib3.disable_warnings() + except (AttributeError, ImportError): + pass + + # Handle IOError issue with VSCode's Terminal (Windows) + click_echo_origin = [click.echo, click.secho] + + def _safe_echo(origin, *args, **kwargs): + try: + click_echo_origin[origin](*args, **kwargs) + except IOError: + (sys.stderr.write if kwargs.get("err") else sys.stdout.write)( + "%s\n" % (args[0] if args else "") + ) + + click.echo = lambda *args, **kwargs: _safe_echo(0, *args, **kwargs) + click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) + + +def main(argv=None): + exit_code = 0 + prev_sys_argv = sys.argv[:] + if argv: + assert isinstance(argv, list) + sys.argv = argv + + try: + ensure_python3(raise_exception=True) + configure() + cli() # pylint: disable=no-value-for-parameter + except SystemExit as exc: + if exc.code and str(exc.code).isdigit(): + exit_code = int(exc.code) + except Exception as exc: # pylint: disable=broad-except + if not isinstance(exc, exception.ReturnErrorCode): + maintenance.on_platformio_exception(exc) + error_str = f"{exc.__class__.__name__}: " + if isinstance(exc, exception.PlatformioException): + error_str += str(exc) + else: + error_str += traceback.format_exc() + error_str += """ +============================================================ + +An unexpected error occurred. Further steps: + +* Verify that you have the latest version of PlatformIO using + `python -m pip install -U platformio` command + +* Try to find answer in FAQ Troubleshooting section + https://docs.platformio.org/page/faq/index.html + +* Report this problem to the developers + https://github.com/platformio/platformio-core/issues + +============================================================ +""" + click.secho(error_str, fg="red", err=True) + exit_code = int(str(exc)) if str(exc).isdigit() else 1 + + maintenance.on_platformio_exit() + sys.argv = prev_sys_argv + return exit_code + + +def debug_gdb_main(): + return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/platformio/account/__init__.py b/platformio/account/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/account/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/account/cli.py b/platformio/account/cli.py new file mode 100644 index 0000000..135445d --- /dev/null +++ b/platformio/account/cli.py @@ -0,0 +1,44 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.commands.destroy import account_destroy_cmd +from platformio.account.commands.forgot import account_forgot_cmd +from platformio.account.commands.login import account_login_cmd +from platformio.account.commands.logout import account_logout_cmd +from platformio.account.commands.password import account_password_cmd +from platformio.account.commands.register import account_register_cmd +from platformio.account.commands.show import account_show_cmd +from platformio.account.commands.token import account_token_cmd +from platformio.account.commands.update import account_update_cmd + + +@click.group( + "account", + commands=[ + account_destroy_cmd, + account_forgot_cmd, + account_login_cmd, + account_logout_cmd, + account_password_cmd, + account_register_cmd, + account_show_cmd, + account_token_cmd, + account_update_cmd, + ], + short_help="Manage PlatformIO account", +) +def cli(): + pass diff --git a/platformio/account/client.py b/platformio/account/client.py new file mode 100644 index 0000000..0643214 --- /dev/null +++ b/platformio/account/client.py @@ -0,0 +1,352 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from platformio import __accounts_api__, app +from platformio.exception import PlatformioException, UserSideException +from platformio.http import HTTPClient, HTTPClientError + + +class AccountError(PlatformioException): + MESSAGE = "{0}" + + +class AccountNotAuthorized(AccountError, UserSideException): + MESSAGE = "You are not authorized! Please log in to PlatformIO Account." + + +class AccountAlreadyAuthorized(AccountError, UserSideException): + MESSAGE = "You are already authorized with {0} account." + + +class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods + SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 + + def __init__(self): + super().__init__(__accounts_api__) + + @staticmethod + def get_refresh_token(): + try: + return app.get_state_item("account").get("auth").get("refresh_token") + except Exception as exc: + raise AccountNotAuthorized() from exc + + @staticmethod + def delete_local_session(): + app.delete_state_item("account") + + @staticmethod + def delete_local_state(key): + account = app.get_state_item("account") + if not account or key not in account: + return + del account[key] + app.set_state_item("account", account) + + def fetch_json_data(self, *args, **kwargs): + try: + return super().fetch_json_data(*args, **kwargs) + except HTTPClientError as exc: + raise AccountError(exc) from exc + + def fetch_authentication_token(self): + if os.environ.get("PLATFORMIO_AUTH_TOKEN"): + return os.environ.get("PLATFORMIO_AUTH_TOKEN") + auth = app.get_state_item("account", {}).get("auth", {}) + if auth.get("access_token") and auth.get("access_token_expire"): + if auth.get("access_token_expire") > time.time(): + return auth.get("access_token") + if auth.get("refresh_token"): + try: + data = self.fetch_json_data( + "post", + "/v1/login", + headers={ + "Authorization": "Bearer %s" % auth.get("refresh_token") + }, + ) + app.set_state_item("account", data) + return data.get("auth").get("access_token") + except AccountError: + self.delete_local_session() + raise AccountNotAuthorized() + + def login(self, username, password): + try: + self.fetch_authentication_token() + except: # pylint:disable=bare-except + pass + else: + raise AccountAlreadyAuthorized( + app.get_state_item("account", {}).get("email", "") + ) + + data = self.fetch_json_data( + "post", + "/v1/login", + data={"username": username, "password": password}, + ) + app.set_state_item("account", data) + return data + + def login_with_code(self, client_id, code, redirect_uri): + try: + self.fetch_authentication_token() + except: # pylint:disable=bare-except + pass + else: + raise AccountAlreadyAuthorized( + app.get_state_item("account", {}).get("email", "") + ) + + result = self.fetch_json_data( + "post", + "/v1/login/code", + data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, + ) + app.set_state_item("account", result) + return result + + def logout(self): + refresh_token = self.get_refresh_token() + self.delete_local_session() + try: + self.fetch_json_data( + "post", + "/v1/logout", + data={"refresh_token": refresh_token}, + ) + except AccountError: + pass + return True + + def change_password(self, old_password, new_password): + return self.fetch_json_data( + "post", + "/v1/password", + data={"old_password": old_password, "new_password": new_password}, + x_with_authorization=True, + ) + + def registration( + self, username, email, password, firstname, lastname + ): # pylint:disable=too-many-arguments + try: + self.fetch_authentication_token() + except: # pylint:disable=bare-except + pass + else: + raise AccountAlreadyAuthorized( + app.get_state_item("account", {}).get("email", "") + ) + + return self.fetch_json_data( + "post", + "/v1/registration", + data={ + "username": username, + "email": email, + "password": password, + "firstname": firstname, + "lastname": lastname, + }, + ) + + def auth_token(self, password, regenerate): + return self.fetch_json_data( + "post", + "/v1/token", + data={"password": password, "regenerate": 1 if regenerate else 0}, + x_with_authorization=True, + ).get("auth_token") + + def forgot_password(self, username): + return self.fetch_json_data( + "post", + "/v1/forgot", + data={"username": username}, + ) + + def get_profile(self): + return self.fetch_json_data( + "get", + "/v1/profile", + x_with_authorization=True, + ) + + def update_profile(self, profile, current_password): + profile["current_password"] = current_password + self.delete_local_state("summary") + response = self.fetch_json_data( + "put", + "/v1/profile", + data=profile, + x_with_authorization=True, + ) + return response + + def get_account_info(self, offline=False): + account = app.get_state_item("account") or {} + if ( + account.get("summary") + and account["summary"].get("expire_at", 0) > time.time() + ): + return account["summary"] + if offline and account.get("email"): + return { + "profile": { + "email": account.get("email"), + "username": account.get("username"), + } + } + result = self.fetch_json_data( + "get", + "/v1/summary", + x_with_authorization=True, + ) + account["summary"] = dict( + profile=result.get("profile"), + packages=result.get("packages"), + subscriptions=result.get("subscriptions"), + user_id=result.get("user_id"), + expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL, + ) + app.set_state_item("account", account) + return result + + def get_logged_username(self): + return self.get_account_info(offline=True).get("profile").get("username") + + def destroy_account(self): + return self.fetch_json_data( + "delete", + "/v1/account", + x_with_authorization=True, + ) + + def create_org(self, orgname, email, displayname): + return self.fetch_json_data( + "post", + "/v1/orgs", + data={"orgname": orgname, "email": email, "displayname": displayname}, + x_with_authorization=True, + ) + + def get_org(self, orgname): + return self.fetch_json_data( + "get", + "/v1/orgs/%s" % orgname, + x_with_authorization=True, + ) + + def list_orgs(self): + return self.fetch_json_data( + "get", + "/v1/orgs", + x_with_authorization=True, + ) + + def update_org(self, orgname, data): + return self.fetch_json_data( + "put", + "/v1/orgs/%s" % orgname, + data={k: v for k, v in data.items() if v}, + x_with_authorization=True, + ) + + def destroy_org(self, orgname): + return self.fetch_json_data( + "delete", + "/v1/orgs/%s" % orgname, + x_with_authorization=True, + ) + + def add_org_owner(self, orgname, username): + return self.fetch_json_data( + "post", + "/v1/orgs/%s/owners" % orgname, + data={"username": username}, + x_with_authorization=True, + ) + + def list_org_owners(self, orgname): + return self.fetch_json_data( + "get", + "/v1/orgs/%s/owners" % orgname, + x_with_authorization=True, + ) + + def remove_org_owner(self, orgname, username): + return self.fetch_json_data( + "delete", + "/v1/orgs/%s/owners" % orgname, + params={"username": username}, + x_with_authorization=True, + ) + + def create_team(self, orgname, teamname, description): + return self.fetch_json_data( + "post", + "/v1/orgs/%s/teams" % orgname, + data={"name": teamname, "description": description}, + x_with_authorization=True, + ) + + def destroy_team(self, orgname, teamname): + return self.fetch_json_data( + "delete", + "/v1/orgs/%s/teams/%s" % (orgname, teamname), + x_with_authorization=True, + ) + + def get_team(self, orgname, teamname): + return self.fetch_json_data( + "get", + "/v1/orgs/%s/teams/%s" % (orgname, teamname), + x_with_authorization=True, + ) + + def list_teams(self, orgname): + return self.fetch_json_data( + "get", + "/v1/orgs/%s/teams" % orgname, + x_with_authorization=True, + ) + + def update_team(self, orgname, teamname, data): + return self.fetch_json_data( + "put", + "/v1/orgs/%s/teams/%s" % (orgname, teamname), + data={k: v for k, v in data.items() if v}, + x_with_authorization=True, + ) + + def add_team_member(self, orgname, teamname, username): + return self.fetch_json_data( + "post", + "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), + data={"username": username}, + x_with_authorization=True, + ) + + def remove_team_member(self, orgname, teamname, username): + return self.fetch_json_data( + "delete", + "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), + params={"username": username}, + x_with_authorization=True, + ) diff --git a/platformio/account/commands/__init__.py b/platformio/account/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/account/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/account/commands/destroy.py b/platformio/account/commands/destroy.py new file mode 100644 index 0000000..5a4f4dd --- /dev/null +++ b/platformio/account/commands/destroy.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient, AccountNotAuthorized + + +@click.command("destroy", short_help="Destroy account") +def account_destroy_cmd(): + client = AccountClient() + click.confirm( + "Are you sure you want to delete the %s user account?\n" + "Warning! All linked data will be permanently removed and can not be restored." + % client.get_logged_username(), + abort=True, + ) + client.destroy_account() + try: + client.logout() + except AccountNotAuthorized: + pass + click.secho( + "User account has been destroyed.", + fg="green", + ) diff --git a/platformio/account/commands/forgot.py b/platformio/account/commands/forgot.py new file mode 100644 index 0000000..4af8f5f --- /dev/null +++ b/platformio/account/commands/forgot.py @@ -0,0 +1,29 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient + + +@click.command("forgot", short_help="Forgot password") +@click.option("--username", prompt="Username or email") +def account_forgot_cmd(username): + client = AccountClient() + client.forgot_password(username) + click.secho( + "If this account is registered, we will send the " + "further instructions to your email.", + fg="green", + ) diff --git a/platformio/account/commands/login.py b/platformio/account/commands/login.py new file mode 100644 index 0000000..b9471bf --- /dev/null +++ b/platformio/account/commands/login.py @@ -0,0 +1,26 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient + + +@click.command("login", short_help="Log in to PlatformIO Account") +@click.option("-u", "--username", prompt="Username or email") +@click.option("-p", "--password", prompt=True, hide_input=True) +def account_login_cmd(username, password): + client = AccountClient() + client.login(username, password) + click.secho("Successfully logged in!", fg="green") diff --git a/platformio/account/commands/logout.py b/platformio/account/commands/logout.py new file mode 100644 index 0000000..243a7f1 --- /dev/null +++ b/platformio/account/commands/logout.py @@ -0,0 +1,24 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient + + +@click.command("logout", short_help="Log out of PlatformIO Account") +def account_logout_cmd(): + client = AccountClient() + client.logout() + click.secho("Successfully logged out!", fg="green") diff --git a/platformio/account/commands/password.py b/platformio/account/commands/password.py new file mode 100644 index 0000000..4f129c0 --- /dev/null +++ b/platformio/account/commands/password.py @@ -0,0 +1,26 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient + + +@click.command("password", short_help="Change password") +@click.option("--old-password", prompt=True, hide_input=True) +@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True) +def account_password_cmd(old_password, new_password): + client = AccountClient() + client.change_password(old_password, new_password) + click.secho("Password successfully changed!", fg="green") diff --git a/platformio/account/commands/register.py b/platformio/account/commands/register.py new file mode 100644 index 0000000..6fc20ce --- /dev/null +++ b/platformio/account/commands/register.py @@ -0,0 +1,52 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.account.validate import ( + validate_email, + validate_password, + validate_username, +) + + +@click.command("register", short_help="Create new PlatformIO Account") +@click.option( + "-u", + "--username", + prompt=True, + callback=lambda _, __, value: validate_username(value), +) +@click.option( + "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value) +) +@click.option( + "-p", + "--password", + prompt=True, + hide_input=True, + confirmation_prompt=True, + callback=lambda _, __, value: validate_password(value), +) +@click.option("--firstname", prompt=True) +@click.option("--lastname", prompt=True) +def account_register_cmd(username, email, password, firstname, lastname): + client = AccountClient() + client.registration(username, email, password, firstname, lastname) + click.secho( + "An account has been successfully created. " + "Please check your mail to activate your account and verify your email address.", + fg="green", + ) diff --git a/platformio/account/commands/show.py b/platformio/account/commands/show.py new file mode 100644 index 0000000..f107f4f --- /dev/null +++ b/platformio/account/commands/show.py @@ -0,0 +1,116 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click +from tabulate import tabulate + +from platformio import util +from platformio.account.client import AccountClient + + +@click.command("show", short_help="PlatformIO Account information") +@click.option("--offline", is_flag=True) +@click.option("--json-output", is_flag=True) +def account_show_cmd(offline, json_output): + client = AccountClient() + info = client.get_account_info(offline) + if json_output: + click.echo(json.dumps(info)) + return + click.echo() + if info.get("profile"): + print_profile(info["profile"]) + if info.get("packages"): + print_packages(info["packages"]) + if info.get("subscriptions"): + print_subscriptions(info["subscriptions"]) + click.echo() + + +def print_profile(profile): + click.secho("Profile", fg="cyan", bold=True) + click.echo("=" * len("Profile")) + data = [] + if profile.get("username"): + data.append(("Username:", profile["username"])) + if profile.get("email"): + data.append(("Email:", profile["email"])) + if profile.get("firstname"): + data.append(("First name:", profile["firstname"])) + if profile.get("lastname"): + data.append(("Last name:", profile["lastname"])) + click.echo(tabulate(data, tablefmt="plain")) + + +def print_packages(packages): + click.echo() + click.secho("Packages", fg="cyan") + click.echo("=" * len("Packages")) + for package in packages: + click.echo() + click.secho(package.get("name"), bold=True) + click.echo("-" * len(package.get("name"))) + if package.get("description"): + click.echo(package.get("description")) + data = [] + expire = "-" + if "subscription" in package: + expire = util.parse_datetime( + package["subscription"].get("end_at") + or package["subscription"].get("next_bill_at") + ).strftime("%Y-%m-%d") + data.append(("Expire:", expire)) + services = [] + for key in package: + if not key.startswith("service."): + continue + if isinstance(package[key], dict): + services.append(package[key].get("title")) + else: + services.append(package[key]) + if services: + data.append(("Services:", ", ".join(services))) + click.echo(tabulate(data, tablefmt="plain")) + + +def print_subscriptions(subscriptions): + click.echo() + click.secho("Subscriptions", fg="cyan") + click.echo("=" * len("Subscriptions")) + for subscription in subscriptions: + click.echo() + click.secho(subscription.get("product_name"), bold=True) + click.echo("-" * len(subscription.get("product_name"))) + data = [("State:", subscription.get("status"))] + begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c") + data.append(("Start date:", begin_at or "-")) + end_at = subscription.get("end_at") + if end_at: + end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c") + data.append(("End date:", end_at or "-")) + next_bill_at = subscription.get("next_bill_at") + if next_bill_at: + next_bill_at = util.parse_datetime( + subscription.get("next_bill_at") + ).strftime("%c") + data.append(("Next payment:", next_bill_at or "-")) + data.append( + ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-") + ) + data.append( + ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-") + ) + click.echo(tabulate(data, tablefmt="plain")) diff --git a/platformio/account/commands/token.py b/platformio/account/commands/token.py new file mode 100644 index 0000000..0f06e36 --- /dev/null +++ b/platformio/account/commands/token.py @@ -0,0 +1,32 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click + +from platformio.account.client import AccountClient + + +@click.command("token", short_help="Get or regenerate Authentication Token") +@click.option("-p", "--password", prompt=True, hide_input=True) +@click.option("--regenerate", is_flag=True) +@click.option("--json-output", is_flag=True) +def account_token_cmd(password, regenerate, json_output): + client = AccountClient() + auth_token = client.auth_token(password, regenerate) + if json_output: + click.echo(json.dumps({"status": "success", "result": auth_token})) + return + click.secho("Personal Authentication Token: %s" % auth_token, fg="green") diff --git a/platformio/account/commands/update.py b/platformio/account/commands/update.py new file mode 100644 index 0000000..b386893 --- /dev/null +++ b/platformio/account/commands/update.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient, AccountNotAuthorized +from platformio.account.validate import validate_email, validate_username + + +@click.command("update", short_help="Update profile information") +@click.option("--current-password", prompt=True, hide_input=True) +@click.option("--username") +@click.option("--email") +@click.option("--firstname") +@click.option("--lastname") +def account_update_cmd(current_password, **kwargs): + client = AccountClient() + profile = client.get_profile() + new_profile = profile.copy() + if not any(kwargs.values()): + for field in profile: + new_profile[field] = click.prompt( + field.replace("_", " ").capitalize(), default=profile[field] + ) + if field == "email": + validate_email(new_profile[field]) + if field == "username": + validate_username(new_profile[field]) + else: + new_profile.update({key: value for key, value in kwargs.items() if value}) + client.update_profile(new_profile, current_password) + click.secho("Profile successfully updated!", fg="green") + username_changed = new_profile["username"] != profile["username"] + email_changed = new_profile["email"] != profile["email"] + if not username_changed and not email_changed: + return None + try: + client.logout() + except AccountNotAuthorized: + pass + if email_changed: + click.secho( + "Please check your mail to verify your new email address and re-login. ", + fg="yellow", + ) + return None + click.secho("Please re-login.", fg="yellow") + return None diff --git a/platformio/account/org/__init__.py b/platformio/account/org/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/account/org/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/account/org/cli.py b/platformio/account/org/cli.py new file mode 100644 index 0000000..5f1f0fc --- /dev/null +++ b/platformio/account/org/cli.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.org.commands.add import org_add_cmd +from platformio.account.org.commands.create import org_create_cmd +from platformio.account.org.commands.destroy import org_destroy_cmd +from platformio.account.org.commands.list import org_list_cmd +from platformio.account.org.commands.remove import org_remove_cmd +from platformio.account.org.commands.update import org_update_cmd + + +@click.group( + "account", + commands=[ + org_add_cmd, + org_create_cmd, + org_destroy_cmd, + org_list_cmd, + org_remove_cmd, + org_update_cmd, + ], + short_help="Manage organizations", +) +def cli(): + pass diff --git a/platformio/account/org/commands/__init__.py b/platformio/account/org/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/account/org/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/account/org/commands/add.py b/platformio/account/org/commands/add.py new file mode 100644 index 0000000..55e658e --- /dev/null +++ b/platformio/account/org/commands/add.py @@ -0,0 +1,34 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient + + +@click.command("add", short_help="Add a new owner to organization") +@click.argument( + "orgname", +) +@click.argument( + "username", +) +def org_add_cmd(orgname, username): + client = AccountClient() + client.add_org_owner(orgname, username) + return click.secho( + "The new owner `%s` has been successfully added to the `%s` organization." + % (username, orgname), + fg="green", + ) diff --git a/platformio/account/org/commands/create.py b/platformio/account/org/commands/create.py new file mode 100644 index 0000000..b0c86bf --- /dev/null +++ b/platformio/account/org/commands/create.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.account.validate import validate_email, validate_orgname + + +@click.command("create", short_help="Create a new organization") +@click.argument( + "orgname", + callback=lambda _, __, value: validate_orgname(value), +) +@click.option( + "--email", callback=lambda _, __, value: validate_email(value) if value else value +) +@click.option( + "--displayname", +) +def org_create_cmd(orgname, email, displayname): + client = AccountClient() + client.create_org(orgname, email, displayname) + return click.secho( + "The organization `%s` has been successfully created." % orgname, + fg="green", + ) diff --git a/platformio/account/org/commands/destroy.py b/platformio/account/org/commands/destroy.py new file mode 100644 index 0000000..2a202f9 --- /dev/null +++ b/platformio/account/org/commands/destroy.py @@ -0,0 +1,34 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient + + +@click.command("destroy", short_help="Destroy organization") +@click.argument("orgname") +def org_destroy_cmd(orgname): + client = AccountClient() + click.confirm( + "Are you sure you want to delete the `%s` organization account?\n" + "Warning! All linked data will be permanently removed and can not be restored." + % orgname, + abort=True, + ) + client.destroy_org(orgname) + return click.secho( + "Organization `%s` has been destroyed." % orgname, + fg="green", + ) diff --git a/platformio/account/org/commands/list.py b/platformio/account/org/commands/list.py new file mode 100644 index 0000000..471814e --- /dev/null +++ b/platformio/account/org/commands/list.py @@ -0,0 +1,48 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click +from tabulate import tabulate + +from platformio.account.client import AccountClient + + +@click.command("list", short_help="List organizations and their members") +@click.option("--json-output", is_flag=True) +def org_list_cmd(json_output): + client = AccountClient() + orgs = client.list_orgs() + if json_output: + return click.echo(json.dumps(orgs)) + if not orgs: + return click.echo("You do not have any organization") + for org in orgs: + click.echo() + click.secho(org.get("orgname"), fg="cyan") + click.echo("-" * len(org.get("orgname"))) + data = [] + if org.get("displayname"): + data.append(("Display Name:", org.get("displayname"))) + if org.get("email"): + data.append(("Email:", org.get("email"))) + data.append( + ( + "Owners:", + ", ".join((owner.get("username") for owner in org.get("owners"))), + ) + ) + click.echo(tabulate(data, tablefmt="plain")) + return click.echo() diff --git a/platformio/account/org/commands/remove.py b/platformio/account/org/commands/remove.py new file mode 100644 index 0000000..9dee462 --- /dev/null +++ b/platformio/account/org/commands/remove.py @@ -0,0 +1,34 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient + + +@click.command("remove", short_help="Remove an owner from organization") +@click.argument( + "orgname", +) +@click.argument( + "username", +) +def org_remove_cmd(orgname, username): + client = AccountClient() + client.remove_org_owner(orgname, username) + return click.secho( + "The `%s` owner has been successfully removed from the `%s` organization." + % (username, orgname), + fg="green", + ) diff --git a/platformio/account/org/commands/update.py b/platformio/account/org/commands/update.py new file mode 100644 index 0000000..86e7cff --- /dev/null +++ b/platformio/account/org/commands/update.py @@ -0,0 +1,50 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.account.validate import validate_email, validate_orgname + + +@click.command("update", short_help="Update organization") +@click.argument("cur_orgname") +@click.option( + "--orgname", + callback=lambda _, __, value: validate_orgname(value) if value else value, + help="A new orgname", +) +@click.option( + "--email", + callback=lambda _, __, value: validate_email(value) if value else value, +) +@click.option("--displayname") +def org_update_cmd(cur_orgname, **kwargs): + client = AccountClient() + org = client.get_org(cur_orgname) + new_org = { + key: value if value is not None else org[key] for key, value in kwargs.items() + } + if not any(kwargs.values()): + for key in kwargs: + new_org[key] = click.prompt(key.capitalize(), default=org[key]) + if key == "email": + validate_email(new_org[key]) + if key == "orgname": + validate_orgname(new_org[key]) + client.update_org(cur_orgname, new_org) + return click.secho( + "The organization `%s` has been successfully updated." % cur_orgname, + fg="green", + ) diff --git a/platformio/account/team/__init__.py b/platformio/account/team/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/account/team/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/account/team/cli.py b/platformio/account/team/cli.py new file mode 100644 index 0000000..feffa7b --- /dev/null +++ b/platformio/account/team/cli.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.team.commands.add import team_add_cmd +from platformio.account.team.commands.create import team_create_cmd +from platformio.account.team.commands.destroy import team_destroy_cmd +from platformio.account.team.commands.list import team_list_cmd +from platformio.account.team.commands.remove import team_remove_cmd +from platformio.account.team.commands.update import team_update_cmd + + +@click.group( + "team", + commands=[ + team_add_cmd, + team_create_cmd, + team_destroy_cmd, + team_list_cmd, + team_remove_cmd, + team_update_cmd, + ], + short_help="Manage organization teams", +) +def cli(): + pass diff --git a/platformio/account/team/commands/__init__.py b/platformio/account/team/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/account/team/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/account/team/commands/add.py b/platformio/account/team/commands/add.py new file mode 100644 index 0000000..854675e --- /dev/null +++ b/platformio/account/team/commands/add.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.account.validate import validate_orgname_teamname + + +@click.command("add", short_help="Add a new member to team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.argument( + "username", +) +def team_add_cmd(orgname_teamname, username): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.add_team_member(orgname, teamname, username) + return click.secho( + "The new member %s has been successfully added to the %s team." + % (username, teamname), + fg="green", + ) diff --git a/platformio/account/team/commands/create.py b/platformio/account/team/commands/create.py new file mode 100644 index 0000000..164ca6a --- /dev/null +++ b/platformio/account/team/commands/create.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.account.validate import validate_orgname_teamname + + +@click.command("create", short_help="Create a new team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.option( + "--description", +) +def team_create_cmd(orgname_teamname, description): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.create_team(orgname, teamname, description) + return click.secho( + "The team %s has been successfully created." % teamname, + fg="green", + ) diff --git a/platformio/account/team/commands/destroy.py b/platformio/account/team/commands/destroy.py new file mode 100644 index 0000000..7cd084a --- /dev/null +++ b/platformio/account/team/commands/destroy.py @@ -0,0 +1,40 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.account.validate import validate_orgname_teamname + + +@click.command("destroy", short_help="Destroy a team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +def team_destroy_cmd(orgname_teamname): + orgname, teamname = orgname_teamname.split(":", 1) + click.confirm( + click.style( + "Are you sure you want to destroy the %s team?" % teamname, fg="yellow" + ), + abort=True, + ) + client = AccountClient() + client.destroy_team(orgname, teamname) + return click.secho( + "The team %s has been successfully destroyed." % teamname, + fg="green", + ) diff --git a/platformio/account/team/commands/list.py b/platformio/account/team/commands/list.py new file mode 100644 index 0000000..d0395d7 --- /dev/null +++ b/platformio/account/team/commands/list.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click +from tabulate import tabulate + +from platformio.account.client import AccountClient + + +@click.command("list", short_help="List teams") +@click.argument("orgname", required=False) +@click.option("--json-output", is_flag=True) +def team_list_cmd(orgname, json_output): + client = AccountClient() + data = {} + if not orgname: + for item in client.list_orgs(): + teams = client.list_teams(item.get("orgname")) + data[item.get("orgname")] = teams + else: + teams = client.list_teams(orgname) + data[orgname] = teams + if json_output: + return click.echo(json.dumps(data[orgname] if orgname else data)) + if not any(data.values()): + return click.secho("You do not have any teams.", fg="yellow") + for org_name, teams in data.items(): + for team in teams: + click.echo() + click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan") + click.echo("-" * len("%s:%s" % (org_name, team.get("name")))) + table_data = [] + if team.get("description"): + table_data.append(("Description:", team.get("description"))) + table_data.append( + ( + "Members:", + ", ".join( + (member.get("username") for member in team.get("members")) + ) + if team.get("members") + else "-", + ) + ) + click.echo(tabulate(table_data, tablefmt="plain")) + return click.echo() diff --git a/platformio/account/team/commands/remove.py b/platformio/account/team/commands/remove.py new file mode 100644 index 0000000..73c7960 --- /dev/null +++ b/platformio/account/team/commands/remove.py @@ -0,0 +1,36 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.account.validate import validate_orgname_teamname + + +@click.command("remove", short_help="Remove a member from team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.argument("username") +def team_remove_cmd(orgname_teamname, username): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.remove_team_member(orgname, teamname, username) + return click.secho( + "The %s member has been successfully removed from the %s team." + % (username, teamname), + fg="green", + ) diff --git a/platformio/account/team/commands/update.py b/platformio/account/team/commands/update.py new file mode 100644 index 0000000..c4c2f8b --- /dev/null +++ b/platformio/account/team/commands/update.py @@ -0,0 +1,51 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.account.validate import validate_orgname_teamname, validate_teamname + + +@click.command("update", short_help="Update team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.option( + "--name", + callback=lambda _, __, value: validate_teamname(value) if value else value, + help="A new team name", +) +@click.option( + "--description", +) +def team_update_cmd(orgname_teamname, **kwargs): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + team = client.get_team(orgname, teamname) + new_team = { + key: value if value is not None else team[key] for key, value in kwargs.items() + } + if not any(kwargs.values()): + for key in kwargs: + new_team[key] = click.prompt(key.capitalize(), default=team[key]) + if key == "name": + validate_teamname(new_team[key]) + client.update_team(orgname, teamname, new_team) + return click.secho( + "The team %s has been successfully updated." % teamname, + fg="green", + ) diff --git a/platformio/account/validate.py b/platformio/account/validate.py new file mode 100644 index 0000000..60d3bb2 --- /dev/null +++ b/platformio/account/validate.py @@ -0,0 +1,84 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import click + + +def validate_username(value, field="username"): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I + ): + raise click.BadParameter( + "Invalid %s format. " + "%s must contain only alphanumeric characters " + "or single hyphens, cannot begin or end with a hyphen, " + "and must not be longer than 38 characters." + % (field.lower(), field.capitalize()) + ) + return value + + +def validate_orgname(value): + return validate_username(value, "Organization name") + + +def validate_email(value): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I + ): + raise click.BadParameter("Invalid email address") + return value + + +def validate_password(value): + value = str(value).strip() if value else None + if not value or not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): + raise click.BadParameter( + "Invalid password format. " + "Password must contain at least 8 characters" + " including a number and a lowercase letter" + ) + return value + + +def validate_teamname(value): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I + ): + raise click.BadParameter( + "Invalid team name format. " + "Team name must only contain alphanumeric characters, " + "single hyphens, underscores, spaces. It can not " + "begin or end with a hyphen or a underscore and must" + " not be longer than 20 characters." + ) + return value + + +def validate_orgname_teamname(value): + value = str(value).strip() if value else None + if not value or ":" not in value: + raise click.BadParameter( + "Please specify organization and team name using the following" + " format - orgname:teamname. For example, mycompany:DreamTeam" + ) + orgname, teamname = value.split(":", 1) + validate_orgname(orgname) + validate_teamname(teamname) + return value diff --git a/platformio/app.py b/platformio/app.py new file mode 100644 index 0000000..7d2f1ea --- /dev/null +++ b/platformio/app.py @@ -0,0 +1,293 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import getpass +import hashlib +import json +import os +import platform +import socket +import time +import uuid + +from platformio import __version__, exception, fs, proc +from platformio.compat import IS_WINDOWS, hashlib_encode_data +from platformio.package.lockfile import LockFile +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_default_projects_dir + + +def projects_dir_validate(projects_dir): + assert os.path.isdir(projects_dir) + return os.path.abspath(projects_dir) + + +DEFAULT_SETTINGS = { + "check_platformio_interval": { + "description": "Check for the new PlatformIO Core interval (days)", + "value": 7, + }, + "check_prune_system_threshold": { + "description": "Check for pruning unnecessary data threshold (megabytes)", + "value": 1024, + }, + "enable_cache": { + "description": "Enable caching for HTTP API requests", + "value": True, + }, + "enable_telemetry": { + "description": ("Telemetry service (Yes/No)"), + "value": True, + }, + "force_verbose": { + "description": "Force verbose output when processing environments", + "value": False, + }, + "projects_dir": { + "description": "Default location for PlatformIO projects (PlatformIO Home)", + "value": get_default_projects_dir(), + "validator": projects_dir_validate, + }, + "enable_proxy_strict_ssl": { + "description": "Verify the proxy server certificate against the list of supplied CAs", + "value": True, + }, +} + +SESSION_VARS = { + "command_ctx": None, + "caller_id": None, + "custom_project_conf": None, + "pause_telemetry": False, +} + + +def resolve_state_path(conf_option_dir, file_name, ensure_dir_exists=True): + state_dir = ProjectConfig.get_instance().get("platformio", conf_option_dir) + if ensure_dir_exists and not os.path.isdir(state_dir): + os.makedirs(state_dir) + return os.path.join(state_dir, file_name) + + +class State: + def __init__(self, path=None, lock=False): + self.path = path + self.lock = lock + if not self.path: + self.path = resolve_state_path("core_dir", "appstate.json") + self._storage = {} + self._lockfile = None + self.modified = False + + def __enter__(self): + try: + self._lock_state_file() + if os.path.isfile(self.path): + self._storage = fs.load_json(self.path) + assert isinstance(self._storage, dict) + except ( + AssertionError, + ValueError, + UnicodeDecodeError, + exception.InvalidJSONFile, + ): + self._storage = {} + return self + + def __exit__(self, type_, value, traceback): + if self.modified: + try: + with open(self.path, mode="w", encoding="utf8") as fp: + fp.write(json.dumps(self._storage)) + except IOError as exc: + raise exception.HomeDirPermissionsError( + os.path.dirname(self.path) + ) from exc + self._unlock_state_file() + + def _lock_state_file(self): + if not self.lock: + return + self._lockfile = LockFile(self.path) + try: + self._lockfile.acquire() + except IOError as exc: + raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) from exc + + def _unlock_state_file(self): + if hasattr(self, "_lockfile") and self._lockfile: + self._lockfile.release() + + def __del__(self): + self._unlock_state_file() + + # Dictionary Proxy + + def as_dict(self): + return self._storage + + def keys(self): + return self._storage.keys() + + def get(self, key, default=True): + return self._storage.get(key, default) + + def update(self, *args, **kwargs): + self.modified = True + return self._storage.update(*args, **kwargs) + + def clear(self): + return self._storage.clear() + + def __getitem__(self, key): + return self._storage[key] + + def __setitem__(self, key, value): + self.modified = True + self._storage[key] = value + + def __delitem__(self, key): + self.modified = True + del self._storage[key] + + def __contains__(self, item): + return item in self._storage + + +def sanitize_setting(name, value): + if name not in DEFAULT_SETTINGS: + raise exception.InvalidSettingName(name) + + defdata = DEFAULT_SETTINGS[name] + try: + if "validator" in defdata: + value = defdata["validator"](value) + elif isinstance(defdata["value"], bool): + if not isinstance(value, bool): + value = str(value).lower() in ("true", "yes", "y", "1") + elif isinstance(defdata["value"], int): + value = int(value) + except Exception as exc: + raise exception.InvalidSettingValue(value, name) from exc + return value + + +def get_state_item(name, default=None): + with State() as state: + return state.get(name, default) + + +def set_state_item(name, value): + with State(lock=True) as state: + state[name] = value + state.modified = True + + +def delete_state_item(name): + with State(lock=True) as state: + if name in state: + del state[name] + + +def get_setting(name): + _env_name = "PLATFORMIO_SETTING_%s" % name.upper() + if _env_name in os.environ: + return sanitize_setting(name, os.getenv(_env_name)) + + with State() as state: + if "settings" in state and name in state["settings"]: + return state["settings"][name] + + return DEFAULT_SETTINGS[name]["value"] + + +def set_setting(name, value): + with State(lock=True) as state: + if "settings" not in state: + state["settings"] = {} + state["settings"][name] = sanitize_setting(name, value) + state.modified = True + + +def reset_settings(): + with State(lock=True) as state: + if "settings" in state: + del state["settings"] + + +def get_session_var(name, default=None): + return SESSION_VARS.get(name, default) + + +def set_session_var(name, value): + assert name in SESSION_VARS + SESSION_VARS[name] = value + + +def is_disabled_progressbar(): + return os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true" + + +def get_cid(): + cid = get_state_item("cid") + if cid: + return cid + uid = None + if os.getenv("GITHUB_USER"): + uid = os.getenv("GITHUB_USER") + elif os.getenv("GITPOD_GIT_USER_NAME"): + uid = os.getenv("GITPOD_GIT_USER_NAME") + if not uid: + uid = uuid.getnode() + cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) + cid = str(cid) + if IS_WINDOWS or os.getuid() > 0: # pylint: disable=no-member + set_state_item("cid", cid) + set_state_item("created_at", int(time.time())) + return cid + + +def get_project_id(project_dir): + return hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest() + + +def get_user_agent(): + data = [ + "PlatformIO/%s" % __version__, + "CI/%d" % int(proc.is_ci()), + "Container/%d" % int(proc.is_container()), + ] + if get_session_var("caller_id"): + data.append("Caller/%s" % get_session_var("caller_id")) + if os.getenv("PLATFORMIO_IDE"): + data.append("IDE/%s" % os.getenv("PLATFORMIO_IDE")) + data.append("Python/%s" % platform.python_version()) + data.append("Platform/%s" % platform.platform()) + if not get_setting("enable_telemetry"): + data.append("Telemetry/0") + return " ".join(data) + + +def get_host_id(): + h = hashlib.sha1(hashlib_encode_data(get_cid())) + try: + username = getpass.getuser() + h.update(hashlib_encode_data(username)) + except: # pylint: disable=bare-except + pass + return h.hexdigest() + + +def get_host_name(): + return str(socket.gethostname())[:255] diff --git a/platformio/assets/schema/library.json b/platformio/assets/schema/library.json new file mode 100644 index 0000000..cd39383 --- /dev/null +++ b/platformio/assets/schema/library.json @@ -0,0 +1,487 @@ +{ + "$id": "https://example.com/library.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "library.json schema", + "type": "object", + "properties": { + "name": { + "type": "string", + "maxLength": 50, + "description": "A name of a library.\nMust be unique in the PlatformIO Registry\nShould be slug style for simplicity, consistency, and compatibility. Example: HelloWorld\nCan contain a-z, digits, and dashes (but not start/end with them)\nConsecutive dashes and [:;/,@<>] chars are not allowed.", + "required": true + }, + "version": { + "type": "string", + "maxLength": 20, + "description": "A version of a current library source code. Can contain a-z, digits, dots or dash and should be Semantic Versioning compatible.", + "required": true + }, + "description": { + "type": "string", + "maxLength": 255, + "description": "The field helps users to identify and search for your library with a brief description. Describe the hardware devices (sensors, boards and etc.) which are suitable with it.", + "required": true + }, + "keywords": { + "anyOf": [ + { + "type": "string", + "maxLength": 255 + }, + { + "type": "array", + "items": { + "type": "string", + "maxLength": 255 + } + } + ], + "description": "Used for search by keyword. Helps to make your library easier to discover without people needing to know its name.\nThe keyword should be lowercased, can contain a-z, digits and dash (but not start/end with them). A list from the keywords can be specified with separator , or declared as Array.", + "required": true + }, + "homepage": { + "type": "string", + "maxLength": 255, + "description": "Home page of a library (if is different from repository url).", + "required": false + }, + "repository": { + "type": "object", + "properties": { + "type": { + "enum": [ + "git", + "hg", + "svn" + ], + "description": "only “git”, “hg” or “svn” are supported" + }, + "url": { + "type": "string" + }, + "branch": { + "type": "string", + "description": "if is not specified, default branch will be used. This field will be ignored if tag/release exists with the value of version." + } + }, + "description": "The repository in which the source code can be found.", + "required": false + }, + "authors": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "required": true, + "description": "Full name" + }, + "email": { + "type": "string" + }, + "url": { + "type": "string", + "description": "An author’s contact page" + }, + "maintainer": { + "type": "boolean", + "description": "Specify “maintainer” status" + } + } + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "required": true, + "description": "Full name" + }, + "email": { + "type": "string" + }, + "url": { + "type": "string", + "description": "An author’s contact page" + }, + "maintainer": { + "type": "boolean", + "description": "Specify “maintainer” status" + } + } + } + } + ], + "description": "An author contact information\nIf authors field is not defined, PlatformIO will try to fetch data from VCS provider (Github, Gitlab, etc) if repository is declared.", + "required": false + }, + "license": { + "type": "string", + "description": "A SPDX license ID or SPDX Expression. You can check the full list of SPDX license IDs (see “Identifier” column).", + "required": false + }, + "frameworks": { + "anyOf": [ + { + "type": "string", + "description": "espidf, freertos, *, etc'" + }, + { + "type": "array", + "items": { + "type": "string", + "description": "espidf, freertos, *, etc'" + } + } + ], + "description": "A list with compatible frameworks. The available framework names are defined in the Frameworks section.\nIf the library is compatible with the all frameworks, then do not declare this field or you use *", + "required": false + }, + "platforms": { + "anyOf": [ + { + "type": "string", + "description": "atmelavr, espressif8266, *, etc'" + }, + { + "type": "array", + "items": { + "type": "string", + "description": "atmelavr, espressif8266, *, etc'" + } + } + ], + "description": "A list with compatible development platforms. The available platform name are defined in Development Platforms section.\nIf the library is compatible with the all platforms, then do not declare this field or use *.\nPlatformIO does not check platforms for compatibility in default mode. See Compatibility Mode for details. If you need a strict checking for compatible platforms for a library, please set libCompatMode to strict.", + "required": false + }, + "headers": { + "anyOf": [ + { + "type": "string", + "description": "MyLibrary.h" + }, + { + "type": "array", + "items": { + "type": "string", + "description": "FooCore.h, FooFeature.h" + } + } + ], + "description": "A list of header files that can be included in a project source files using #include <...> directive.", + "required": false + }, + "examples": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "base": { + "type": "string" + }, + "files": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "description": "A list of example patterns.", + "required": "false" + }, + "dependencies": { + "anyOf": [ + { + "type": "object", + "properties": { + "owner": { + "type": "string", + "description": "an owner name (username) from the PlatformIO Registry" + }, + "name": { + "type": "string", + "description": "library name" + }, + "version": { + "type": "string", + "description": "Version Requirements or Package Specifications" + }, + "frameworks": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "project compatible Frameworks" + }, + "platforms": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": " project compatible Development Platforms" + } + } + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "owner": { + "type": "string", + "description": "an owner name (username) from the PlatformIO Registry" + }, + "name": { + "type": "string", + "description": "library name" + }, + "version": { + "type": "string", + "description": "Version Requirements or Package Specifications" + }, + "frameworks": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "project compatible Frameworks" + }, + "platforms": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": " project compatible Development Platforms" + } + } + } + } + ], + "description": "A list of dependent libraries that will be automatically installed.", + "required": false + }, + "export": { + "type": "object", + "properties": { + "include": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Export only files that matched declared patterns.\n* - matches everything\n? - matches any single character\n[seq] - matches any character in seq\n[!seq] - matches any character not in seq" + }, + "exclude": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Exclude the directories and files which match with exclude patterns." + } + }, + "description": "This option is useful if you need to exclude extra data (test code, docs, images, PDFs, etc). It allows one to reduce the size of the final archive.\nTo check which files will be included in the final packages, please use pio pkg pack command.", + "required": false + }, + "scripts": { + "type": "object", + "properties": { + "postinstall": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "runs a script AFTER the package has been installed.\nRun a custom Python script located in the package “scripts” folder AFTER the package is installed. Please note that you don’t need to specify a Python interpreter for Python scripts" + }, + "preuninstall": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "runs a script BEFORE the package is removed.\nRun a custom Bash script BEFORE the package is uninstalled. The script is declared as a list of command arguments and is located at the root of a package" + } + }, + "description": "Execute custom scripts during the special Package Management CLI life cycle events", + "required": false + }, + "build": { + "type": "object", + "properties": { + "flags": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "Extra flags to control preprocessing, compilation, assembly, and linking processes. More details build_flags.\nKeep in mind when operating with the -I flag (directories to be searched for header files). The path should be relative to the root directory where the library.json manifest is located." + }, + "unflags": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "Remove base/initial flags which were set by development platform. More details build_unflags." + }, + "includeDir": { + "type": "string", + "description": "Custom directory to be searched for header files. A default value is include and means that folder is located at the root of a library.\nThe Library Dependency Finder (LDF) will pick a library automatically only when a project or other dependent libraries include any header file located in includeDir or srcDir.", + "required": false + }, + "srcDir": { + "type": "string", + "description": "Custom location of library source code. A default value is src and means that folder is located in the root of a library.", + "required": "false" + }, + "srcFilter": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "Specify which source files should be included/excluded from build process. The path in filter should be relative to the srcDir option of a library.\nSee syntax for build_src_filter.\nPlease note that you can generate source filter “on-the-fly” using extraScript", + "required": false + }, + "extraScript": { + "type": "string", + "description": "Launch extra script before a build process.", + "required": "false" + }, + "libArchive": { + "type": "boolean", + "description": "Create an archive (*.a, static library) from the object files and link it into a firmware (program). This is default behavior of PlatformIO Build System (\"libArchive\": true).\nSetting \"libArchive\": false will instruct PlatformIO Build System to link object files directly (in-line). This could be useful if you need to override weak symbols defined in framework or other libraries.\nYou can disable library archiving globally using lib_archive option in “platformio.ini” (Project Configuration File).", + "required": "false" + }, + "libLDFMode": { + "anyOf": [ + { + "enum": [ + "off" + ], + "description": "“Manual mode”, does not process source files of a project and dependencies. Builds only the libraries that are specified in manifests (library.json, module.json) or using lib_deps option." + }, + { + "enum": [ + "chain" + ], + "description": "[DEFAULT] Parses ALL C/C++ source files of the project and follows only by nested includes (#include ..., chain...) from the libraries. It also parses C, CC, CPP files from libraries which have the same name as included header file. Does not evaluate C/C++ Preprocessor conditional syntax." + }, + { + "enum": [ + "deep" + ], + "description": "Parses ALL C/C++ source files of the project and parses ALL C/C++ source files of the each found dependency (recursively). Does not evaluate C/C++ Preprocessor conditional syntax." + }, + { + "enum": [ + "chain+" + ], + "description": "The same behavior as for the chain but evaluates C/C++ Preprocessor conditional syntax." + }, + { + "enum": [ + "deep+" + ], + "description": "The same behavior as for the deep but evaluates C/C++ Preprocessor conditional syntax." + } + ], + "description": "Specify Library Dependency Finder Mode. See Dependency Finder Mode for details.", + "required": false + }, + "libCompatMode": { + "type": "string", + "description": "Specify Library Compatibility Mode. See Compatibility Mode for details.", + "required": false + }, + "builder": { + "anyOf": [ + { + "enum": [ + "PlatformIOLibBuilder" + ], + "description": "Default Builder" + }, + { + "enum": [ + "ArduinoLibBuilder" + ] + }, + { + "enum": [ + "MbedLibBuilder" + ] + } + ], + "description": "Override default PlatformIOLibBuilder with another builder.", + "required": false + } + }, + "required": false + } + } +} \ No newline at end of file diff --git a/platformio/assets/system/99-platformio-udev.rules b/platformio/assets/system/99-platformio-udev.rules new file mode 100644 index 0000000..a5897f8 --- /dev/null +++ b/platformio/assets/system/99-platformio-udev.rules @@ -0,0 +1,173 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##################################################################################### +# +# INSTALLATION +# +# Please visit > https://docs.platformio.org/en/latest/core/installation/udev-rules.html +# +##################################################################################### + +# +# Boards +# + +# CP210X USB UART +ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea[67][013]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="80a9", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# FT231XS USB UART +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Prolific Technology, Inc. PL2303 Serial Port +ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# QinHeng Electronics HL-340 USB-Serial adapter +ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +# QinHeng Electronics CH9102 USB-Serial adapter +ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d4", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Arduino boards +ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][023]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="[08][02]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Arduino SAM-BA +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="6124", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{MTP_NO_PROBE}="1" + +# Digistump boards +ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Maple with DFU +ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# USBtiny +ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# USBasp V2.0 +ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Teensy boards +ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1" +SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MODE:="0666" +KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666" + +# TI Stellaris Launchpad +ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# TI MSP430 Launchpad +ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# GD32V DFU Bootloader +ATTRS{idVendor}=="28e9", ATTRS{idProduct}=="0189", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# FireBeetle-ESP32 +ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7522", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Wio Terminal +ATTRS{idVendor}=="2886", ATTRS{idProduct}=="[08]02d", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Raspberry Pi Pico +ATTRS{idVendor}=="2e8a", ATTRS{idProduct}=="[01]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# AIR32F103 +ATTRS{idVendor}=="0d28", ATTRS{idProduct}=="0204", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + + +# +# Debuggers +# + +# Black Magic Probe +SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# opendous and estick +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Original FT232/FT245/FT2232/FT232H/FT4232 +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="60[01][104]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# DISTORTEC JTAG-lock-pick Tiny 2 +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# TUMPA, TUMPA Lite +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a9[89]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# XDS100v2 +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE) +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca[01]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# TI/Luminary Stellaris Evaluation Board FTDI (several) +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd[9a]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# egnite Turtelizer 2 +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Section5 ICEbear +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c14[01]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Amontec JTAGkey and JTAGkey-tiny +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# TI ICDI +ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# STLink probes +ATTRS{idVendor}=="0483", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Hilscher NXHX Boards +ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Hitex probes +ATTRS{idVendor}=="0640", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Altera USB Blaster +ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Amontec JTAGkey-HiSpeed +ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# SEGGER J-Link +ATTRS{idVendor}=="1366", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Raisonance RLink +ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Debug Board for Neo1973 +ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Olimex probes +ATTRS{idVendor}=="15ba", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# USBprog with OpenOCD firmware +ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board +ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Marvell Sheevaplug +ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Keil Software, Inc. ULink +ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# CMSIS-DAP compatible adapters +ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Atmel AVR Dragon +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2107", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" diff --git a/platformio/builder/__init__.py b/platformio/builder/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/builder/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/builder/main.py b/platformio/builder/main.py new file mode 100644 index 0000000..ea8b202 --- /dev/null +++ b/platformio/builder/main.py @@ -0,0 +1,250 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import sys +from time import time + +import click +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import DEFAULT_TARGETS # pylint: disable=import-error +from SCons.Script import AllowSubstExceptions # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error +from SCons.Script import Default # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error +from SCons.Script import Import # pylint: disable=import-error +from SCons.Script import Variables # pylint: disable=import-error + +from platformio import app, fs +from platformio.platform.base import PlatformBase +from platformio.proc import get_pythonexe_path +from platformio.project.helpers import get_project_dir + +AllowSubstExceptions(NameError) + +# append CLI arguments to build environment +clivars = Variables(None) +clivars.AddVariables( + ("BUILD_SCRIPT",), + ("PROJECT_CONFIG",), + ("PIOENV",), + ("PIOTEST_RUNNING_NAME",), + ("UPLOAD_PORT",), + ("PROGRAM_ARGS",), +) + +DEFAULT_ENV_OPTIONS = dict( + tools=[ + "ar", + "cc", + "c++", + "link", + "piohooks", + "pioasm", + "piobuild", + "pioproject", + "pioplatform", + "piotest", + "piotarget", + "piolib", + "pioupload", + "piosize", + "pioino", + "piomisc", + "piointegration", + "piomaxlen", + ], + toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")], + variables=clivars, + # Propagating External Environment + ENV=os.environ, + UNIX_TIME=int(time()), + BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"), + BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"), + BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"), + COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"), + LIBPATH=["$BUILD_DIR"], + PROGNAME="program", + PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), + PROG_PATH="$PROGPATH", # deprecated + PYTHONEXE=get_pythonexe_path(), +) + +# Declare command verbose messages +command_strings = dict( + ARCOM="Archiving", + LINKCOM="Linking", + RANLIBCOM="Indexing", + ASCOM="Compiling", + ASPPCOM="Compiling", + CCCOM="Compiling", + CXXCOM="Compiling", +) +if not int(ARGUMENTS.get("PIOVERBOSE", 0)): + for name, value in command_strings.items(): + DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value) + +env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) +env.SConscriptChdir(False) + +# Load variables from CLI +env.Replace( + **{ + key: PlatformBase.decode_scons_arg(env[key]) + for key in list(clivars.keys()) + if key in env + } +) + +# Setup project optional directories +config = env.GetProjectConfig() +app.set_session_var("custom_project_conf", config.path) + +env.Replace( + PROJECT_DIR=get_project_dir(), + PROJECT_CORE_DIR=config.get("platformio", "core_dir"), + PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"), + PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"), + PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"), + PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"), + PROJECT_SRC_DIR=config.get("platformio", "src_dir"), + PROJECTSRC_DIR="$PROJECT_SRC_DIR", # legacy for dev/platform + PROJECT_TEST_DIR=config.get("platformio", "test_dir"), + PROJECT_DATA_DIR=config.get("platformio", "data_dir"), + PROJECTDATA_DIR="$PROJECT_DATA_DIR", # legacy for dev/platform + PROJECT_BUILD_DIR=config.get("platformio", "build_dir"), + BUILD_TYPE=env.GetBuildType(), + BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"), + LIBSOURCE_DIRS=[ + config.get("platformio", "lib_dir"), + os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), + config.get("platformio", "globallib_dir"), + ], +) + +if int(ARGUMENTS.get("ISATTY", 0)): + # pylint: disable=protected-access + click._compat.isatty = lambda stream: True + +if env.subst("$BUILD_CACHE_DIR"): + if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")): + os.makedirs(env.subst("$BUILD_CACHE_DIR")) + env.CacheDir("$BUILD_CACHE_DIR") + +if not int(ARGUMENTS.get("PIOVERBOSE", 0)): + click.echo("Verbose mode can be enabled via `-v, --verbose` option") + +# Dynamically load dependent tools +if "compiledb" in COMMAND_LINE_TARGETS: + env.Tool("compilation_db") + +if not os.path.isdir(env.subst("$BUILD_DIR")): + os.makedirs(env.subst("$BUILD_DIR")) + +env.LoadProjectOptions() +env.LoadPioPlatform() + +env.SConsignFile( + os.path.join( + "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR", + ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]), + ) +) + +env.SConscript(env.GetExtraScripts("pre"), exports="env") + +if env.IsCleanTarget(): + env.CleanProject(fullclean=int(ARGUMENTS.get("FULLCLEAN", 0))) + env.Exit(0) + +env.SConscript("$BUILD_SCRIPT") + +if "UPLOAD_FLAGS" in env: + env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"]) +if env.GetProjectOption("upload_command"): + env.Replace(UPLOADCMD=env.GetProjectOption("upload_command")) + +env.SConscript(env.GetExtraScripts("post"), exports="env") + +############################################################################## + +# Checking program size +if env.get("SIZETOOL") and not ( + set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS) +): + env.Depends("upload", "checkprogsize") + # Replace platform's "size" target with our + _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"] + Default(None) + Default(_new_targets) + Default("checkprogsize") + +if "compiledb" in COMMAND_LINE_TARGETS: + env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH")) + +# Print configured protocols +env.AddPreAction( + "upload", + env.VerboseAction( + lambda source, target, env: env.PrintUploadInfo(), + "Configuring upload protocol...", + ), +) + +AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS)) +AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) + +env.ProcessDelayedActions() + +############################################################################## + +if "envdump" in COMMAND_LINE_TARGETS: + click.echo(env.Dump()) + env.Exit(0) + +if env.IsIntegrationDump(): + projenv = None + try: + Import("projenv") + except: # pylint: disable=bare-except + projenv = env + data = projenv.DumpIntegrationData(env) + # dump to file for the further reading by project.helpers.load_build_metadata + with open( + projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")), + mode="w", + encoding="utf8", + ) as fp: + json.dump(data, fp) + click.echo("\n%s\n" % json.dumps(data)) # pylint: disable=undefined-variable + env.Exit(0) + +if "sizedata" in COMMAND_LINE_TARGETS: + AlwaysBuild( + env.Alias( + "sizedata", + DEFAULT_TARGETS, + env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."), + ) + ) + + Default("sizedata") + +# issue #4604: process targets sequentially +for index, target in enumerate( + [t for t in COMMAND_LINE_TARGETS if not t.startswith("__")][1:] +): + env.Depends(target, COMMAND_LINE_TARGETS[index]) diff --git a/platformio/builder/tools/__init__.py b/platformio/builder/tools/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/builder/tools/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/builder/tools/pioasm.py b/platformio/builder/tools/pioasm.py new file mode 100644 index 0000000..4d5cfc1 --- /dev/null +++ b/platformio/builder/tools/pioasm.py @@ -0,0 +1,29 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import SCons.Tool.asm # pylint: disable=import-error + +# +# Resolve https://github.com/platformio/platformio-core/issues/3917 +# Avoid forcing .S to bare assembly on Windows OS +# + +if ".S" in SCons.Tool.asm.ASSuffixes: + SCons.Tool.asm.ASSuffixes.remove(".S") +if ".S" not in SCons.Tool.asm.ASPPSuffixes: + SCons.Tool.asm.ASPPSuffixes.append(".S") + + +generate = SCons.Tool.asm.generate +exists = SCons.Tool.asm.exists diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py new file mode 100644 index 0000000..74725a0 --- /dev/null +++ b/platformio/builder/tools/piobuild.py @@ -0,0 +1,390 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import os +import sys + +from SCons import Builder, Util # pylint: disable=import-error +from SCons.Node import FS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error +from SCons.Script import SConscript # pylint: disable=import-error + +from platformio import __version__, fs +from platformio.compat import IS_MACOS, string_types +from platformio.package.version import pepver_to_semver +from platformio.proc import where_is_program + +SRC_HEADER_EXT = ["h", "hpp"] +SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] +SRC_C_EXT = ["c"] +SRC_CXX_EXT = ["cc", "cpp", "cxx", "c++"] +SRC_BUILD_EXT = SRC_C_EXT + SRC_CXX_EXT + SRC_ASM_EXT +SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % os.sep, "-<.svn%s>" % os.sep] + + +def scons_patched_match_splitext(path, suffixes=None): + """Patch SCons Builder, append $OBJSUFFIX to the end of each target""" + tokens = Util.splitext(path) + if suffixes and tokens[1] and tokens[1] in suffixes: + return (path, tokens[1]) + return tokens + + +def GetBuildType(env): + modes = [] + if ( + set(["__debug", "sizedata"]) # sizedata = for memory inspection + & set(COMMAND_LINE_TARGETS) + or env.GetProjectOption("build_type") == "debug" + ): + modes.append("debug") + if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test": + modes.append("test") + return "+".join(modes or ["release"]) + + +def BuildProgram(env): + env.ProcessProgramDeps() + env.ProcessProjectDeps() + + # append into the beginning a main LD script + if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): + env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) + + # enable "cyclic reference" for linker + if ( + env.get("LIBS") + and env.GetCompilerType() == "gcc" + and (env.PioPlatform().is_embedded() or not IS_MACOS) + ): + env.Prepend(_LIBFLAGS="-Wl,--start-group ") + env.Append(_LIBFLAGS=" -Wl,--end-group") + + program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"]) + env.Replace(PIOMAINPROG=program) + + AlwaysBuild( + env.Alias( + "checkprogsize", + program, + env.VerboseAction(env.CheckUploadSize, "Checking size $PIOMAINPROG"), + ) + ) + + print("Building in %s mode" % env["BUILD_TYPE"]) + + return program + + +def ProcessProgramDeps(env): + def _append_pio_macros(): + core_version = pepver_to_semver(__version__) + env.AppendUnique( + CPPDEFINES=[ + ( + "PLATFORMIO", + int( + "{0:02d}{1:02d}{2:02d}".format( + core_version.major, core_version.minor, core_version.patch + ) + ), + ) + ] + ) + + _append_pio_macros() + + env.PrintConfiguration() + + # process extra flags from board + if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): + env.ProcessFlags(env.BoardConfig().get("build.extra_flags")) + + # apply user flags + env.ProcessFlags(env.get("BUILD_FLAGS")) + + # process framework scripts + env.BuildFrameworks(env.get("PIOFRAMEWORK")) + + if "debug" in env["BUILD_TYPE"]: + env.ConfigureDebugTarget() + + # remove specified flags + env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) + + if "compiledb" in COMMAND_LINE_TARGETS: + # Resolve absolute path of toolchain + for cmd in ("CC", "CXX", "AS"): + if cmd not in env: + continue + if os.path.isabs(env[cmd]): + continue + env[cmd] = where_is_program( + env.subst("$%s" % cmd), env.subst("${ENV['PATH']}") + ) + + if env.get("COMPILATIONDB_INCLUDE_TOOLCHAIN"): + for scope, includes in env.DumpIntegrationIncludes().items(): + if scope in ("toolchain",): + env.Append(CPPPATH=includes) + + +def ProcessProjectDeps(env): + plb = env.ConfigureProjectLibBuilder() + + # prepend project libs to the beginning of list + env.Prepend(LIBS=plb.build()) + # prepend extra linker related options from libs + env.PrependUnique( + **{ + key: plb.env.get(key) + for key in ("LIBS", "LIBPATH", "LINKFLAGS") + if plb.env.get(key) + } + ) + + if "test" in env["BUILD_TYPE"]: + build_files_before_nums = len(env.get("PIOBUILDFILES", [])) + plb.env.BuildSources( + "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" + ) + if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1: + sys.stderr.write( + "Error: Nothing to build. Please put your test suites " + "to the '%s' folder\n" % env.subst("$PROJECT_TEST_DIR") + ) + env.Exit(1) + + if "test" not in env["BUILD_TYPE"] or env.GetProjectOption("test_build_src"): + plb.env.BuildSources( + "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") + ) + + if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: + sys.stderr.write( + "Error: Nothing to build. Please put your source code files " + "to the '%s' folder\n" % env.subst("$PROJECT_SRC_DIR") + ) + env.Exit(1) + + +def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches + if not isinstance(flags, list): + flags = [flags] + result = {} + for raw in flags: + for key, value in env.ParseFlags(str(raw)).items(): + if key not in result: + result[key] = [] + result[key].extend(value) + + cppdefines = [] + for item in result["CPPDEFINES"]: + if not Util.is_Sequence(item): + cppdefines.append(item) + continue + name, value = item[:2] + if '"' in value: + value = value.replace('"', '\\"') + elif value.isdigit(): + value = int(value) + elif value.replace(".", "", 1).isdigit(): + value = float(value) + cppdefines.append((name, value)) + result["CPPDEFINES"] = cppdefines + + # fix relative CPPPATH & LIBPATH + for k in ("CPPPATH", "LIBPATH"): + for i, p in enumerate(result.get(k, [])): + p = env.subst(p) + if os.path.isdir(p): + result[k][i] = os.path.abspath(p) + + # fix relative path for "-include" + for i, f in enumerate(result.get("CCFLAGS", [])): + if isinstance(f, tuple) and f[0] == "-include": + result["CCFLAGS"][i] = (f[0], env.subst(f[1].get_path())) + + return result + + +def ProcessFlags(env, flags): # pylint: disable=too-many-branches + if not flags: + return + env.Append(**env.ParseFlagsExtended(flags)) + + # Cancel any previous definition of name, either built in or + # provided with a -U option // Issue #191 + undefines = [ + u + for u in env.get("CCFLAGS", []) + if isinstance(u, string_types) and u.startswith("-U") + ] + if undefines: + for undef in undefines: + env["CCFLAGS"].remove(undef) + if undef[2:] in env["CPPDEFINES"]: + env["CPPDEFINES"].remove(undef[2:]) + env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines)) + + +def ProcessUnFlags(env, flags): + if not flags: + return + parsed = env.ParseFlagsExtended(flags) + unflag_scopes = tuple(set(["ASPPFLAGS"] + list(parsed.keys()))) + for scope in unflag_scopes: + for unflags in parsed.values(): + for unflag in unflags: + for current in list(env.get(scope, [])): + conditions = [ + unflag == current, + not isinstance(unflag, (tuple, list)) + and isinstance(current, (tuple, list)) + and unflag == current[0], + ] + if any(conditions): + env[scope].remove(current) + + +def StringifyMacro(env, value): # pylint: disable=unused-argument + return '\\"%s\\"' % value.replace('"', '\\\\\\"') + + +def MatchSourceFiles(env, src_dir, src_filter=None, src_exts=None): + src_filter = env.subst(src_filter) if src_filter else None + src_filter = src_filter or SRC_FILTER_DEFAULT + src_exts = src_exts or (SRC_BUILD_EXT + SRC_HEADER_EXT) + return fs.match_src_files(env.subst(src_dir), src_filter, src_exts) + + +def CollectBuildFiles( + env, variant_dir, src_dir, src_filter=None, duplicate=False +): # pylint: disable=too-many-locals + sources = [] + variants = [] + + src_dir = env.subst(src_dir) + if src_dir.endswith(os.sep): + src_dir = src_dir[:-1] + + for item in env.MatchSourceFiles(src_dir, src_filter, SRC_BUILD_EXT): + _reldir = os.path.dirname(item) + _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir + _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir + + if _var_dir not in variants: + variants.append(_var_dir) + env.VariantDir(_var_dir, _src_dir, duplicate) + + sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) + + middlewares = env.get("__PIO_BUILD_MIDDLEWARES") + if not middlewares: + return sources + + new_sources = [] + for node in sources: + new_node = node + for callback, pattern in middlewares: + if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): + continue + if callback.__code__.co_argcount == 2: + new_node = callback(env, new_node) + else: + new_node = callback(new_node) + if not new_node: + break + if new_node: + new_sources.append(new_node) + + return new_sources + + +def AddBuildMiddleware(env, callback, pattern=None): + env.Append(__PIO_BUILD_MIDDLEWARES=[(callback, pattern)]) + + +def BuildFrameworks(env, frameworks): + if not frameworks: + return + + if "BOARD" not in env: + sys.stderr.write( + "Please specify `board` in `platformio.ini` to use " + "with '%s' framework\n" % ", ".join(frameworks) + ) + env.Exit(1) + + supported_frameworks = env.BoardConfig().get("frameworks", []) + for name in frameworks: + if name == "arduino": + # Arduino IDE appends .o to the end of filename + Builder.match_splitext = scons_patched_match_splitext + if "nobuild" not in COMMAND_LINE_TARGETS: + env.ConvertInoToCpp() + + if name in supported_frameworks: + SConscript(env.GetFrameworkScript(name), exports="env") + else: + sys.stderr.write("Error: This board doesn't support %s framework!\n" % name) + env.Exit(1) + + +def BuildLibrary(env, variant_dir, src_dir, src_filter=None, nodes=None): + env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) + nodes = nodes or env.CollectBuildFiles(variant_dir, src_dir, src_filter) + return env.StaticLibrary(env.subst(variant_dir), nodes) + + +def BuildSources(env, variant_dir, src_dir, src_filter=None): + if env.get("PIOMAINPROG"): + sys.stderr.write( + "Error: The main program is already constructed and the inline " + "source files are not allowed. Please use `env.BuildLibrary(...)` " + "or PRE-type script instead." + ) + env.Exit(1) + + nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter) + DefaultEnvironment().Append( + PIOBUILDFILES=[ + env.Object(node) if isinstance(node, FS.File) else node for node in nodes + ] + ) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(GetBuildType) + env.AddMethod(BuildProgram) + env.AddMethod(ProcessProgramDeps) + env.AddMethod(ProcessProjectDeps) + env.AddMethod(ParseFlagsExtended) + env.AddMethod(ProcessFlags) + env.AddMethod(ProcessUnFlags) + env.AddMethod(StringifyMacro) + env.AddMethod(MatchSourceFiles) + env.AddMethod(CollectBuildFiles) + env.AddMethod(AddBuildMiddleware) + env.AddMethod(BuildFrameworks) + env.AddMethod(BuildLibrary) + env.AddMethod(BuildSources) + return env diff --git a/platformio/builder/tools/piohooks.py b/platformio/builder/tools/piohooks.py new file mode 100644 index 0000000..9c67b00 --- /dev/null +++ b/platformio/builder/tools/piohooks.py @@ -0,0 +1,50 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def AddActionWrapper(handler): + def wraps(env, files, action): + if not isinstance(files, (list, tuple, set)): + files = [files] + known_nodes = [] + unknown_files = [] + for item in files: + nodes = env.arg2nodes(item, env.fs.Entry) + if nodes and nodes[0].exists(): + known_nodes.extend(nodes) + else: + unknown_files.append(item) + if unknown_files: + env.Append(**{"_PIO_DELAYED_ACTIONS": [(handler, unknown_files, action)]}) + if known_nodes: + return handler(known_nodes, action) + return [] + + return wraps + + +def ProcessDelayedActions(env): + for func, nodes, action in env.get("_PIO_DELAYED_ACTIONS", []): + func(nodes, action) + + +def generate(env): + env.Replace(**{"_PIO_DELAYED_ACTIONS": []}) + env.AddMethod(AddActionWrapper(env.AddPreAction), "AddPreAction") + env.AddMethod(AddActionWrapper(env.AddPostAction), "AddPostAction") + env.AddMethod(ProcessDelayedActions) + + +def exists(_): + return True diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py new file mode 100644 index 0000000..c44e72b --- /dev/null +++ b/platformio/builder/tools/pioino.py @@ -0,0 +1,256 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +import glob +import io +import os +import re +import tempfile + +import click + +from platformio.compat import get_filesystem_encoding, get_locale_encoding + + +class InoToCPPConverter: + PROTOTYPE_RE = re.compile( + r"""^( + (?:template\<.*\>\s*)? # template + ([a-z_\d\&]+\*?\s+){1,2} # return type + ([a-z_\d]+\s*) # name of prototype + \([a-z_,\.\*\&\[\]\s\d]*\) # arguments + )\s*(\{|;) # must end with `{` or `;` + """, + re.X | re.M | re.I, + ) + DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) + PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)" + + def __init__(self, env): + self.env = env + self._main_ino = None + self._safe_encoding = None + + def read_safe_contents(self, path): + error_reported = False + for encoding in ( + "utf-8", + None, + get_filesystem_encoding(), + get_locale_encoding(), + "latin-1", + ): + try: + with io.open(path, encoding=encoding) as fp: + contents = fp.read() + self._safe_encoding = encoding + return contents + except UnicodeDecodeError: + if not error_reported: + error_reported = True + click.secho( + "Unicode decode error has occurred, please remove invalid " + "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" + % path, + fg="yellow", + err=True, + ) + return "" + + def write_safe_contents(self, path, contents): + with io.open( + path, "w", encoding=self._safe_encoding, errors="backslashreplace" + ) as fp: + return fp.write(contents) + + def is_main_node(self, contents): + return self.DETECTMAIN_RE.search(contents) + + def convert(self, nodes): + contents = self.merge(nodes) + if not contents: + return None + return self.process(contents) + + def merge(self, nodes): + assert nodes + lines = [] + for node in nodes: + contents = self.read_safe_contents(node.get_path()) + _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] + if self.is_main_node(contents): + lines = _lines + lines + self._main_ino = node.get_path() + else: + lines.extend(_lines) + + if not self._main_ino: + self._main_ino = nodes[0].get_path() + + return "\n".join(["#include "] + lines) if lines else None + + def process(self, contents): + out_file = re.sub(r"[\"\'\;]+", "", self._main_ino) + ".cpp" + assert self._gcc_preprocess(contents, out_file) + contents = self.read_safe_contents(out_file) + contents = self._join_multiline_strings(contents) + self.write_safe_contents(out_file, self.append_prototypes(contents)) + return out_file + + def _gcc_preprocess(self, contents, out_file): + tmp_path = tempfile.mkstemp()[1] + self.write_safe_contents(tmp_path, contents) + self.env.Execute( + self.env.VerboseAction( + '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( + out_file, tmp_path + ), + "Converting " + os.path.basename(out_file[:-4]), + ) + ) + atexit.register(_delete_file, tmp_path) + return os.path.isfile(out_file) + + def _join_multiline_strings(self, contents): + if "\\\n" not in contents: + return contents + newlines = [] + linenum = 0 + stropen = False + for line in contents.split("\n"): + _linenum = self._parse_preproc_line_num(line) + if _linenum is not None: + linenum = _linenum + else: + linenum += 1 + + if line.endswith("\\"): + if line.startswith('"'): + stropen = True + newlines.append(line[:-1]) + continue + if stropen: + newlines[len(newlines) - 1] += line[:-1] + continue + elif stropen and line.endswith(('",', '";')): + newlines[len(newlines) - 1] += line + stropen = False + newlines.append( + '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/")) + ) + continue + + newlines.append(line) + + return "\n".join(newlines) + + @staticmethod + def _parse_preproc_line_num(line): + if not line.startswith("#"): + return None + tokens = line.split(" ", 3) + if len(tokens) > 2 and tokens[1].isdigit(): + return int(tokens[1]) + return None + + def _parse_prototypes(self, contents): + prototypes = [] + reserved_keywords = set(["if", "else", "while"]) + for match in self.PROTOTYPE_RE.finditer(contents): + if ( + set([match.group(2).strip(), match.group(3).strip()]) + & reserved_keywords + ): + continue + prototypes.append(match) + return prototypes + + def _get_total_lines(self, contents): + total = 0 + if contents.endswith("\n"): + contents = contents[:-1] + for line in contents.split("\n")[::-1]: + linenum = self._parse_preproc_line_num(line) + if linenum is not None: + return total + linenum + total += 1 + return total + + def append_prototypes(self, contents): + prototypes = self._parse_prototypes(contents) or [] + + # skip already declared prototypes + declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";") + prototypes = [m for m in prototypes if m.group(1).strip() not in declared] + + if not prototypes: + return contents + + prototype_names = set(m.group(3).strip() for m in prototypes) + split_pos = prototypes[0].start() + match_ptrs = re.search( + self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), + contents[:split_pos], + re.M, + ) + if match_ptrs: + split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1 + + result = [] + result.append(contents[:split_pos].strip()) + result.append("%s;" % ";\n".join([m.group(1) for m in prototypes])) + result.append( + '#line %d "%s"' + % ( + self._get_total_lines(contents[:split_pos]), + self._main_ino.replace("\\", "/"), + ) + ) + result.append(contents[split_pos:].strip()) + return "\n".join(result) + + +def FindInoNodes(env): + src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR")) + return env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( + os.path.join(src_dir, "*.pde") + ) + + +def ConvertInoToCpp(env): + ino_nodes = env.FindInoNodes() + if not ino_nodes: + return + c = InoToCPPConverter(env) + out_file = c.convert(ino_nodes) + + atexit.register(_delete_file, out_file) + + +def _delete_file(path): + try: + if os.path.isfile(path): + os.remove(path) + except: # pylint: disable=bare-except + pass + + +def generate(env): + env.AddMethod(FindInoNodes) + env.AddMethod(ConvertInoToCpp) + + +def exists(_): + return True diff --git a/platformio/builder/tools/piointegration.py b/platformio/builder/tools/piointegration.py new file mode 100644 index 0000000..6f0a972 --- /dev/null +++ b/platformio/builder/tools/piointegration.py @@ -0,0 +1,193 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os + +import SCons.Defaults # pylint: disable=import-error +import SCons.Subst # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error + +from platformio.proc import exec_command, where_is_program + + +def IsIntegrationDump(_): + return set(["__idedata", "idedata"]) & set(COMMAND_LINE_TARGETS) + + +def DumpIntegrationIncludes(env): + result = dict(build=[], compatlib=[], toolchain=[]) + + result["build"].extend( + [ + env.subst("$PROJECT_INCLUDE_DIR"), + env.subst("$PROJECT_SRC_DIR"), + ] + ) + result["build"].extend( + [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])] + ) + + # installed libs + for lb in env.GetLibBuilders(): + result["compatlib"].extend( + [os.path.abspath(inc) for inc in lb.get_include_dirs()] + ) + + # includes from toolchains + p = env.PioPlatform() + for pkg in p.get_installed_packages(with_optional=False): + if p.get_package_type(pkg.metadata.name) != "toolchain": + continue + toolchain_dir = glob.escape(pkg.path) + toolchain_incglobs = [ + os.path.join(toolchain_dir, "*", "include", "c++", "*"), + os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), + os.path.join(toolchain_dir, "lib", "gcc", "*", "*", "include*"), + os.path.join(toolchain_dir, "*", "include*"), + ] + for g in toolchain_incglobs: + result["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)]) + + return result + + +def get_gcc_defines(env): + items = [] + try: + sysenv = os.environ.copy() + sysenv["PATH"] = str(env["ENV"]["PATH"]) + result = exec_command( + "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True + ) + except OSError: + return items + if result["returncode"] != 0: + return items + for line in result["out"].split("\n"): + tokens = line.strip().split(" ", 2) + if not tokens or tokens[0] != "#define": + continue + if len(tokens) > 2: + items.append("%s=%s" % (tokens[1], tokens[2])) + else: + items.append(tokens[1]) + return items + + +def dump_defines(env): + defines = [] + # global symbols + for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])): + item = item.strip() + if item: + defines.append(env.subst(item).replace('\\"', '"')) + + # special symbol for Atmel AVR MCU + if env["PIOPLATFORM"] == "atmelavr": + board_mcu = env.get("BOARD_MCU") + if not board_mcu and "BOARD" in env: + board_mcu = env.BoardConfig().get("build.mcu") + if board_mcu: + defines.append( + str( + "__AVR_%s__" + % board_mcu.upper() + .replace("ATMEGA", "ATmega") + .replace("ATTINY", "ATtiny") + ) + ) + + # built-in GCC marcos + # if env.GetCompilerType() == "gcc": + # defines.extend(get_gcc_defines(env)) + + return defines + + +def dump_svd_path(env): + svd_path = env.GetProjectOption("debug_svd_path") + if svd_path: + return os.path.abspath(svd_path) + + if "BOARD" not in env: + return None + try: + svd_path = env.BoardConfig().get("debug.svd_path") + assert svd_path + except (AssertionError, KeyError): + return None + # custom path to SVD file + if os.path.isfile(svd_path): + return svd_path + # default file from ./platform/misc/svd folder + p = env.PioPlatform() + if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)): + return os.path.abspath(os.path.join(p.get_dir(), "misc", "svd", svd_path)) + return None + + +def _split_flags_string(env, s): + args = env.subst_list(s, SCons.Subst.SUBST_CMD)[0] + return [str(arg) for arg in args] + + +def DumpIntegrationData(*args): + projenv, globalenv = args[0:2] # pylint: disable=unbalanced-tuple-unpacking + data = { + "build_type": globalenv.GetBuildType(), + "env_name": globalenv["PIOENV"], + "libsource_dirs": [ + globalenv.subst(item) for item in globalenv.GetLibSourceDirs() + ], + "defines": dump_defines(projenv), + "includes": projenv.DumpIntegrationIncludes(), + "cc_flags": _split_flags_string(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS"), + "cxx_flags": _split_flags_string(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS"), + "cc_path": where_is_program( + globalenv.subst("$CC"), globalenv.subst("${ENV['PATH']}") + ), + "cxx_path": where_is_program( + globalenv.subst("$CXX"), globalenv.subst("${ENV['PATH']}") + ), + "gdb_path": where_is_program( + globalenv.subst("$GDB"), globalenv.subst("${ENV['PATH']}") + ), + "prog_path": globalenv.subst("$PROGPATH"), + "svd_path": dump_svd_path(globalenv), + "compiler_type": globalenv.GetCompilerType(), + "targets": globalenv.DumpTargets(), + "extra": dict( + flash_images=[ + {"offset": item[0], "path": globalenv.subst(item[1])} + for item in globalenv.get("FLASH_EXTRA_IMAGES", []) + ] + ), + } + for key in ("IDE_EXTRA_DATA", "INTEGRATION_EXTRA_DATA"): + data["extra"].update(globalenv.get(key, {})) + return data + + +def exists(_): + return True + + +def generate(env): + env["IDE_EXTRA_DATA"] = {} # legacy support + env["INTEGRATION_EXTRA_DATA"] = {} + env.AddMethod(IsIntegrationDump) + env.AddMethod(DumpIntegrationIncludes) + env.AddMethod(DumpIntegrationData) + return env diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py new file mode 100644 index 0000000..dee1d5b --- /dev/null +++ b/platformio/builder/tools/piolib.py @@ -0,0 +1,1209 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-instance-attributes, too-many-public-methods +# pylint: disable=assignment-from-no-return, unused-argument, too-many-lines + +import hashlib +import io +import os +import re +import sys + +import click +import SCons.Scanner # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error + +from platformio import exception, fs +from platformio.builder.tools import piobuild +from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types +from platformio.http import HTTPClientError, InternetConnectionError +from platformio.package.exception import ( + MissingPackageManifestError, + UnknownPackageError, +) +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manifest.parser import ( + ManifestParserError, + ManifestParserFactory, +) +from platformio.package.meta import PackageCompatibility, PackageItem +from platformio.project.options import ProjectOptions + + +class LibBuilderFactory: + @staticmethod + def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): + clsname = "UnknownLibBuilder" + if os.path.isfile(os.path.join(path, "library.json")): + clsname = "PlatformIOLibBuilder" + else: + used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) + common_frameworks = set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks) + if common_frameworks: + clsname = "%sLibBuilder" % list(common_frameworks)[0].capitalize() + elif used_frameworks: + clsname = "%sLibBuilder" % used_frameworks[0].capitalize() + + obj = globals()[clsname](env, path, verbose=verbose) + + # Handle PlatformIOLibBuilder.manifest.build.builder + # pylint: disable=protected-access + if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get( + "builder" + ): + obj = globals()[obj._manifest.get("build", {}).get("builder")]( + env, path, verbose=verbose + ) + + assert isinstance(obj, LibBuilderBase) + return obj + + @staticmethod + def get_used_frameworks(env, path): + if any( + os.path.isfile(os.path.join(path, fname)) + for fname in ("library.properties", "keywords.txt") + ): + return ["arduino"] + + if os.path.isfile(os.path.join(path, "module.json")): + return ["mbed"] + + include_re = re.compile( + r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', flags=re.MULTILINE + ) + + # check source files + for root, _, files in os.walk(path, followlinks=True): + if "mbed_lib.json" in files: + return ["mbed"] + for fname in files: + if not fs.path_endswith_ext( + fname, piobuild.SRC_BUILD_EXT + piobuild.SRC_HEADER_EXT + ): + continue + with io.open( + os.path.join(root, fname), encoding="utf8", errors="ignore" + ) as fp: + content = fp.read() + if not content: + continue + if "Arduino.h" in content and include_re.search(content): + return ["arduino"] + if "mbed.h" in content and include_re.search(content): + return ["mbed"] + return [] + + +class LibBuilderBase: + CLASSIC_SCANNER = SCons.Scanner.C.CScanner() + CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner() + # Max depth of nested includes: + # -1 = unlimited + # 0 - disabled nesting + # >0 - number of allowed nested includes + CCONDITIONAL_SCANNER_DEPTH = 99 + PARSE_SRC_BY_H_NAME = True + + _INCLUDE_DIRS_CACHE = None + + def __init__(self, env, path, manifest=None, verbose=False): + self.env = env.Clone() + self.envorigin = env.Clone() + self.path = os.path.abspath(env.subst(path)) + self.verbose = verbose + + try: + self._manifest = manifest if manifest else self.load_manifest() + except ManifestParserError: + click.secho( + "Warning! Ignoring broken library manifest in " + self.path, fg="yellow" + ) + self._manifest = {} + + self.is_dependent = False + self.is_built = False + self.depbuilders = [] + + self._deps_are_processed = False + self._circular_deps = [] + self._processed_search_files = [] + + # pass a macro to the projenv + libs + if "test" in env["BUILD_TYPE"]: + self.env.Append(CPPDEFINES=["PIO_UNIT_TESTING"]) + + # reset source filter, could be overridden with extra script + self.env["SRC_FILTER"] = "" + + # process extra options and append to build environment + self.process_extra_options() + + def __repr__(self): + return "%s(%r)" % (self.__class__, self.path) + + def __contains__(self, child_path): + return self.is_common_builder(self.path, child_path) + + def is_common_builder(self, root_path, child_path): + if IS_WINDOWS: + root_path = root_path.lower() + child_path = child_path.lower() + if root_path == child_path: + return True + if ( + os.path.commonprefix([root_path + os.path.sep, child_path]) + == root_path + os.path.sep + ): + return True + # try to resolve paths + root_path = os.path.realpath(root_path) + child_path = os.path.realpath(child_path) + return ( + os.path.commonprefix([root_path + os.path.sep, child_path]) + == root_path + os.path.sep + ) + + @property + def name(self): + return self._manifest.get("name", os.path.basename(self.path)) + + @property + def version(self): + return self._manifest.get("version") + + @property + def dependent(self): + """Backward compatibility with ESP-IDF""" + return self.is_dependent + + @property + def dependencies(self): + return self._manifest.get("dependencies") + + @property + def src_filter(self): + return piobuild.SRC_FILTER_DEFAULT + [ + "-" % os.sep, + "-" % os.sep, + "-" % os.sep, + "-" % os.sep, + ] + + @property + def include_dir(self): + for name in ("include", "Include"): + d = os.path.join(self.path, name) + if os.path.isdir(d): + return d + return None + + @property + def src_dir(self): + for name in ("src", "Src"): + d = os.path.join(self.path, name) + if os.path.isdir(d): + return d + return self.path + + def get_include_dirs(self): + items = [] + include_dir = self.include_dir + if include_dir: + items.append(include_dir) + items.append(self.src_dir) + return items + + @property + def build_dir(self): + lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] + return os.path.join( + "$BUILD_DIR", "lib%s" % lib_hash, os.path.basename(self.path) + ) + + @property + def build_flags(self): + return None + + @property + def build_unflags(self): + return None + + @property + def extra_script(self): + return None + + @property + def lib_archive(self): + return self.env.GetProjectOption("lib_archive") + + @property + def lib_ldf_mode(self): + return self.env.GetProjectOption("lib_ldf_mode") + + @staticmethod + def validate_ldf_mode(mode): + ldf_modes = ProjectOptions["env.lib_ldf_mode"].type.choices + if isinstance(mode, string_types): + mode = mode.strip().lower() + if mode in ldf_modes: + return mode + try: + return ldf_modes[int(mode)] + except (IndexError, ValueError): + pass + return ProjectOptions["env.lib_ldf_mode"].default + + @property + def lib_compat_mode(self): + return self.env.GetProjectOption("lib_compat_mode") + + @staticmethod + def validate_compat_mode(mode): + compat_modes = ProjectOptions["env.lib_compat_mode"].type.choices + if isinstance(mode, string_types): + mode = mode.strip().lower() + if mode in compat_modes: + return mode + try: + return compat_modes[int(mode)] + except (IndexError, ValueError): + pass + return ProjectOptions["env.lib_compat_mode"].default + + def is_platforms_compatible(self, platforms): + return True + + def is_frameworks_compatible(self, frameworks): + return True + + def load_manifest(self): + return {} + + def process_extra_options(self): + with fs.cd(self.path): + self.env.ProcessFlags(self.build_flags) + if self.extra_script: + self.env.SConscriptChdir(True) + self.env.SConscript( + os.path.abspath(self.extra_script), + exports={"env": self.env, "pio_lib_builder": self}, + ) + self.env.SConscriptChdir(False) + self.env.ProcessUnFlags(self.build_unflags) + + def process_dependencies(self): + if not self.dependencies or self._deps_are_processed: + return + self._deps_are_processed = True + for item in self.dependencies: + found = False + for lb in self.env.GetLibBuilders(): + if item["name"] != lb.name: + continue + found = True + if lb not in self.depbuilders: + self.depend_on(lb) + break + + if not found and self.verbose: + sys.stderr.write( + "Warning: Ignored `%s` dependency for `%s` " + "library\n" % (item["name"], self.name) + ) + + def get_search_files(self): + return [ + os.path.join(self.src_dir, item) + for item in self.env.MatchSourceFiles( + self.src_dir, self.src_filter, piobuild.SRC_BUILD_EXT + ) + ] + + def get_implicit_includes( # pylint: disable=too-many-branches + self, search_files=None + ): + # all include directories + if not LibBuilderBase._INCLUDE_DIRS_CACHE: + LibBuilderBase._INCLUDE_DIRS_CACHE = [ + self.env.Dir(d) + for d in ProjectAsLibBuilder( + self.envorigin, "$PROJECT_DIR", export_projenv=False + ).get_include_dirs() + ] + for lb in self.env.GetLibBuilders(): + LibBuilderBase._INCLUDE_DIRS_CACHE.extend( + [self.env.Dir(d) for d in lb.get_include_dirs()] + ) + + # append self include directories + include_dirs = [self.env.Dir(d) for d in self.get_include_dirs()] + include_dirs.extend(LibBuilderBase._INCLUDE_DIRS_CACHE) + + result = [] + search_files = search_files or [] + while search_files: + node = self.env.File(search_files.pop(0)) + if node.get_abspath() in self._processed_search_files: + continue + self._processed_search_files.append(node.get_abspath()) + + try: + assert "+" in self.lib_ldf_mode + candidates = LibBuilderBase.CCONDITIONAL_SCANNER( + node, + self.env, + tuple(include_dirs), + depth=self.CCONDITIONAL_SCANNER_DEPTH, + ) + + except Exception as exc: # pylint: disable=broad-except + if self.verbose and "+" in self.lib_ldf_mode: + sys.stderr.write( + "Warning! Classic Pre Processor is used for `%s`, " + "advanced has failed with `%s`\n" % (node.get_abspath(), exc) + ) + candidates = LibBuilderBase.CLASSIC_SCANNER( + node, self.env, tuple(include_dirs) + ) + + # print(node.get_abspath(), [c.get_abspath() for c in candidates]) + for item in candidates: + item_path = item.get_abspath() + # process internal files recursively + if ( + item_path not in self._processed_search_files + and item_path not in search_files + and item_path in self + ): + search_files.append(item_path) + if item not in result: + result.append(item) + if not self.PARSE_SRC_BY_H_NAME: + continue + if not fs.path_endswith_ext(item_path, piobuild.SRC_HEADER_EXT): + continue + item_fname = item_path[: item_path.rindex(".")] + for ext in piobuild.SRC_C_EXT + piobuild.SRC_CXX_EXT: + if not os.path.isfile("%s.%s" % (item_fname, ext)): + continue + item_c_node = self.env.File("%s.%s" % (item_fname, ext)) + if item_c_node not in result: + result.append(item_c_node) + + return result + + def search_deps_recursive(self, search_files=None): + self.process_dependencies() + + # when LDF is disabled + if self.lib_ldf_mode == "off": + return + + if self.lib_ldf_mode.startswith("deep"): + search_files = self.get_search_files() + + lib_inc_map = {} + for inc in self.get_implicit_includes(search_files): + inc_path = inc.get_abspath() + for lb in self.env.GetLibBuilders(): + if inc_path in lb: + if lb not in lib_inc_map: + lib_inc_map[lb] = [] + lib_inc_map[lb].append(inc_path) + break + + for lb, lb_search_files in lib_inc_map.items(): + self.depend_on(lb, search_files=lb_search_files) + + def depend_on(self, lb, search_files=None, recursive=True): + def _already_depends(_lb): + if self in _lb.depbuilders: + return True + for __lb in _lb.depbuilders: + if _already_depends(__lb): + return True + return False + + # assert isinstance(lb, LibBuilderBase) + if self != lb: + if _already_depends(lb): + if self.verbose: + sys.stderr.write( + "Warning! Circular dependencies detected " + "between `%s` and `%s`\n" % (self.path, lb.path) + ) + self._circular_deps.append(lb) + elif lb not in self.depbuilders: + self.depbuilders.append(lb) + lb.is_dependent = True + LibBuilderBase._INCLUDE_DIRS_CACHE = None + + if recursive: + lb.search_deps_recursive(search_files) + + def build(self): + libs = [] + shared_scopes = ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS") + for lb in self.depbuilders: + libs.extend(lb.build()) + # copy shared information to self env + self.env.PrependUnique( + **{ + scope: lb.env.get(scope) + for scope in shared_scopes + if lb.env.get(scope) + } + ) + + for lb in self._circular_deps: + self.env.PrependUnique(CPPPATH=lb.get_include_dirs()) + + if self.is_built: + return libs + self.is_built = True + + self.env.PrependUnique(CPPPATH=self.get_include_dirs()) + + if self.lib_ldf_mode == "off": + for lb in self.env.GetLibBuilders(): + if self == lb or not lb.is_built: + continue + self.env.PrependUnique( + **{ + scope: lb.env.get(scope) + for scope in shared_scopes + if lb.env.get(scope) + } + ) + + do_not_archive = not self.lib_archive + if not do_not_archive: + nodes = self.env.CollectBuildFiles( + self.build_dir, self.src_dir, self.src_filter + ) + if nodes: + libs.append( + self.env.BuildLibrary( + self.build_dir, self.src_dir, self.src_filter, nodes + ) + ) + else: + do_not_archive = True + if do_not_archive: + self.env.BuildSources(self.build_dir, self.src_dir, self.src_filter) + + return libs + + +class UnknownLibBuilder(LibBuilderBase): + pass + + +class ArduinoLibBuilder(LibBuilderBase): + def load_manifest(self): + manifest_path = os.path.join(self.path, "library.properties") + if not os.path.isfile(manifest_path): + return {} + return ManifestParserFactory.new_from_file(manifest_path).as_dict() + + @property + def include_dir(self): + if not all( + os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src") + ): + return None + return os.path.join(self.path, "include") + + def get_include_dirs(self): + include_dirs = super().get_include_dirs() + if os.path.isdir(os.path.join(self.path, "src")): + return include_dirs + if os.path.isdir(os.path.join(self.path, "utility")): + include_dirs.append(os.path.join(self.path, "utility")) + return include_dirs + + @property + def src_filter(self): + src_dir = os.path.join(self.path, "src") + if os.path.isdir(src_dir): + # pylint: disable=no-member + src_filter = LibBuilderBase.src_filter.fget(self) + for root, _, files in os.walk(src_dir, followlinks=True): + found = False + for fname in files: + if fname.lower().endswith("asm"): + found = True + break + if not found: + continue + rel_path = root.replace(src_dir, "") + if rel_path.startswith(os.path.sep): + rel_path = rel_path[1:] + os.path.sep + src_filter.append("-<%s*.[aA][sS][mM]>" % rel_path) + return src_filter + + src_filter = [] + is_utility = os.path.isdir(os.path.join(self.path, "utility")) + for ext in piobuild.SRC_BUILD_EXT + piobuild.SRC_HEADER_EXT: + # arduino ide ignores files with .asm or .ASM extensions + if ext.lower() == "asm": + continue + src_filter.append("+<*.%s>" % ext) + if is_utility: + src_filter.append("+" % (os.path.sep, ext)) + return src_filter + + @property + def dependencies(self): + # do not include automatically all libraries for build + # chain+ will decide later + return None + + @property + def lib_ldf_mode(self): + # pylint: disable=no-member + if not self._manifest.get("dependencies"): + return LibBuilderBase.lib_ldf_mode.fget(self) + missing = object() + global_value = self.env.GetProjectConfig().getraw( + "env:" + self.env["PIOENV"], "lib_ldf_mode", missing + ) + if global_value != missing: + return LibBuilderBase.lib_ldf_mode.fget(self) + # automatically enable C++ Preprocessing in runtime + # (Arduino IDE has this behavior) + return "chain+" + + def is_frameworks_compatible(self, frameworks): + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=["arduino", "energia"]) + ) + + def is_platforms_compatible(self, platforms): + return PackageCompatibility(platforms=platforms).is_compatible( + PackageCompatibility(platforms=self._manifest.get("platforms")) + ) + + @property + def build_flags(self): + ldflags = [ + LibBuilderBase.build_flags.fget(self), # pylint: disable=no-member + self._manifest.get("ldflags"), + ] + if self._manifest.get("precompiled") in ("true", "full"): + # add to LDPATH {build.mcu} folder + board_config = self.env.BoardConfig() + for key in ("build.mcu", "build.cpu"): + libpath = os.path.join(self.src_dir, board_config.get(key, "")) + if not os.path.isdir(libpath): + continue + self.env.PrependUnique(LIBPATH=libpath) + break + ldflags = [flag for flag in ldflags if flag] # remove empty + return " ".join(ldflags) if ldflags else None + + +class MbedLibBuilder(LibBuilderBase): + def load_manifest(self): + manifest_path = os.path.join(self.path, "module.json") + if not os.path.isfile(manifest_path): + return {} + return ManifestParserFactory.new_from_file(manifest_path).as_dict() + + @property + def src_dir(self): + if os.path.isdir(os.path.join(self.path, "source")): + return os.path.join(self.path, "source") + return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member + + def get_include_dirs(self): + include_dirs = super().get_include_dirs() + if self.path not in include_dirs: + include_dirs.append(self.path) + + # library with module.json + for p in self._manifest.get("extraIncludes", []): + include_dirs.append(os.path.join(self.path, p)) + + # old mbed library without manifest, add to CPPPATH all folders + if not self._manifest: + for root, _, __ in os.walk(self.path): + part = root.replace(self.path, "").lower() + if any(s in part for s in ("%s." % os.path.sep, "test", "example")): + continue + if root not in include_dirs: + include_dirs.append(root) + + return include_dirs + + def is_frameworks_compatible(self, frameworks): + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=["mbed"]) + ) + + def process_extra_options(self): + self._process_mbed_lib_confs() + return super().process_extra_options() + + def _process_mbed_lib_confs(self): + mbed_lib_paths = [ + os.path.join(root, "mbed_lib.json") + for root, _, files in os.walk(self.path) + if "mbed_lib.json" in files + ] + if not mbed_lib_paths: + return None + + mbed_config_path = None + for p in self.env.get("CPPPATH"): + mbed_config_path = os.path.join(self.env.subst(p), "mbed_config.h") + if os.path.isfile(mbed_config_path): + break + mbed_config_path = None + if not mbed_config_path: + return None + + macros = {} + for mbed_lib_path in mbed_lib_paths: + macros.update(self._mbed_lib_conf_parse_macros(mbed_lib_path)) + + self._mbed_conf_append_macros(mbed_config_path, macros) + return True + + @staticmethod + def _mbed_normalize_macro(macro): + name = macro + value = None + if "=" in macro: + name, value = macro.split("=", 1) + return dict(name=name, value=value) + + def _mbed_lib_conf_parse_macros(self, mbed_lib_path): + macros = {} + cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES"))) + manifest = fs.load_json(mbed_lib_path) + + # default macros + for macro in manifest.get("macros", []): + macro = self._mbed_normalize_macro(macro) + macros[macro["name"]] = macro + + # configuration items + for key, options in manifest.get("config", {}).items(): + if "value" not in options: + continue + macros[key] = dict( + name=options.get("macro_name"), value=options.get("value") + ) + + # overrode items per target + for target, options in manifest.get("target_overrides", {}).items(): + if target != "*" and "TARGET_" + target not in cppdefines: + continue + for macro in options.get("target.macros_add", []): + macro = self._mbed_normalize_macro(macro) + macros[macro["name"]] = macro + for key, value in options.items(): + if not key.startswith("target.") and key in macros: + macros[key]["value"] = value + + # normalize macro names + for key, macro in macros.items(): + if not macro["name"]: + macro["name"] = key + if "." not in macro["name"]: + macro["name"] = "%s.%s" % (manifest.get("name"), macro["name"]) + macro["name"] = re.sub( + r"[^a-z\d]+", "_", macro["name"], flags=re.I + ).upper() + macro["name"] = "MBED_CONF_" + macro["name"] + if isinstance(macro["value"], bool): + macro["value"] = 1 if macro["value"] else 0 + + return {macro["name"]: macro["value"] for macro in macros.values()} + + def _mbed_conf_append_macros(self, mbed_config_path, macros): + lines = [] + with open(mbed_config_path, encoding="utf8") as fp: + for line in fp.readlines(): + line = line.strip() + if line == "#endif": + lines.append("// PlatformIO Library Dependency Finder (LDF)") + lines.extend( + [ + "#define %s %s" % (name, value if value is not None else "") + for name, value in macros.items() + ] + ) + lines.append("") + if not line.startswith("#define"): + lines.append(line) + continue + tokens = line.split() + if len(tokens) < 2 or tokens[1] not in macros: + lines.append(line) + lines.append("") + with open(mbed_config_path, mode="w", encoding="utf8") as fp: + fp.write("\n".join(lines)) + + +class PlatformIOLibBuilder(LibBuilderBase): + def load_manifest(self): + manifest_path = os.path.join(self.path, "library.json") + if not os.path.isfile(manifest_path): + return {} + return ManifestParserFactory.new_from_file(manifest_path).as_dict() + + def _has_arduino_manifest(self): + return os.path.isfile(os.path.join(self.path, "library.properties")) + + @property + def include_dir(self): + if "includeDir" in self._manifest.get("build", {}): + with fs.cd(self.path): + return os.path.abspath(self._manifest.get("build").get("includeDir")) + return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member + + def get_include_dirs(self): + include_dirs = super().get_include_dirs() + + # backwards compatibility with PlatformIO 2.0 + if ( + "build" not in self._manifest + and self._has_arduino_manifest() + and not os.path.isdir(os.path.join(self.path, "src")) + and os.path.isdir(os.path.join(self.path, "utility")) + ): + include_dirs.append(os.path.join(self.path, "utility")) + + for path in self.env.get("CPPPATH", []): + if path not in self.envorigin.get("CPPPATH", []): + include_dirs.append(self.env.subst(path)) + + return include_dirs + + @property + def src_dir(self): + if "srcDir" in self._manifest.get("build", {}): + with fs.cd(self.path): + return os.path.abspath(self._manifest.get("build").get("srcDir")) + return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member + + @property + def src_filter(self): + # pylint: disable=no-member + if "srcFilter" in self._manifest.get("build", {}): + return self._manifest.get("build").get("srcFilter") + if self.env["SRC_FILTER"]: + return self.env["SRC_FILTER"] + if self._has_arduino_manifest(): + return ArduinoLibBuilder.src_filter.fget(self) + return LibBuilderBase.src_filter.fget(self) + + @property + def build_flags(self): + if "flags" in self._manifest.get("build", {}): + return self._manifest.get("build").get("flags") + return LibBuilderBase.build_flags.fget(self) # pylint: disable=no-member + + @property + def build_unflags(self): + if "unflags" in self._manifest.get("build", {}): + return self._manifest.get("build").get("unflags") + return LibBuilderBase.build_unflags.fget(self) # pylint: disable=no-member + + @property + def extra_script(self): + if "extraScript" in self._manifest.get("build", {}): + return self._manifest.get("build").get("extraScript") + return LibBuilderBase.extra_script.fget(self) # pylint: disable=no-member + + @property + def lib_archive(self): + missing = object() + global_value = self.env.GetProjectConfig().getraw( + "env:" + self.env["PIOENV"], "lib_archive", missing + ) + if global_value != missing: + return self.env.GetProjectConfig().get( + "env:" + self.env["PIOENV"], "lib_archive" + ) + # pylint: disable=no-member + return self._manifest.get("build", {}).get( + "libArchive", LibBuilderBase.lib_archive.fget(self) + ) + + @property + def lib_ldf_mode(self): + # pylint: disable=no-member + return self.validate_ldf_mode( + self._manifest.get("build", {}).get( + "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self) + ) + ) + + @property + def lib_compat_mode(self): + # pylint: disable=no-member + return self.validate_compat_mode( + self._manifest.get("build", {}).get( + "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self) + ) + ) + + def is_platforms_compatible(self, platforms): + return PackageCompatibility(platforms=platforms).is_compatible( + PackageCompatibility(platforms=self._manifest.get("platforms")) + ) + + def is_frameworks_compatible(self, frameworks): + return PackageCompatibility(frameworks=frameworks).is_compatible( + PackageCompatibility(frameworks=self._manifest.get("frameworks")) + ) + + +class ProjectAsLibBuilder(LibBuilderBase): + def __init__(self, env, *args, **kwargs): + export_projenv = kwargs.get("export_projenv", True) + if "export_projenv" in kwargs: + del kwargs["export_projenv"] + # backup original value, will be reset in base.__init__ + project_src_filter = env.get("SRC_FILTER") + super().__init__(env, *args, **kwargs) + self.env["SRC_FILTER"] = project_src_filter + if export_projenv: + env.Export(dict(projenv=self.env)) + + def __contains__(self, child_path): + for root_path in (self.include_dir, self.src_dir, self.test_dir): + if root_path and self.is_common_builder(root_path, child_path): + return True + return False + + @property + def include_dir(self): + include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") + return include_dir if os.path.isdir(include_dir) else None + + @property + def src_dir(self): + return self.env.subst("$PROJECT_SRC_DIR") + + @property + def test_dir(self): + return self.env.subst("$PROJECT_TEST_DIR") + + def get_search_files(self): + items = [] + build_type = self.env["BUILD_TYPE"] + # project files + if "test" not in build_type or self.env.GetProjectOption("test_build_src"): + items.extend(super().get_search_files()) + # test files + if "test" in build_type: + items.extend( + [ + os.path.join("$PROJECT_TEST_DIR", item) + for item in self.env.MatchSourceFiles( + "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" + ) + ] + ) + return items + + @property + def lib_ldf_mode(self): + mode = LibBuilderBase.lib_ldf_mode.fget(self) # pylint: disable=no-member + if not mode.startswith("chain"): + return mode + # parse all project files + return "deep+" if "+" in mode else "deep" + + @property + def src_filter(self): + # pylint: disable=no-member + return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self) + + @property + def build_flags(self): + # pylint: disable=no-member + return self.env.get("SRC_BUILD_FLAGS") or LibBuilderBase.build_flags.fget(self) + + @property + def dependencies(self): + return self.env.GetProjectOption("lib_deps", []) + + def process_extra_options(self): + with fs.cd(self.path): + self.env.ProcessFlags(self.build_flags) + self.env.ProcessUnFlags(self.build_unflags) + + def install_dependencies(self): + def _is_builtin(spec): + for lb in self.env.GetLibBuilders(): + if lb.name == spec: + return True + return False + + not_found_specs = [] + for spec in self.dependencies: + # check if built-in library + if _is_builtin(spec): + continue + + found = False + for storage_dir in self.env.GetLibSourceDirs(): + lm = LibraryPackageManager(storage_dir) + if lm.get_package(spec): + found = True + break + if not found: + not_found_specs.append(spec) + + did_install = False + lm = LibraryPackageManager( + self.env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) + ) + for spec in not_found_specs: + try: + lm.install(spec) + did_install = True + except ( + HTTPClientError, + UnknownPackageError, + InternetConnectionError, + ) as exc: + click.secho("Warning! %s" % exc, fg="yellow") + + # reset cache + if did_install: + DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) + + def process_dependencies(self): # pylint: disable=too-many-branches + found_lbs = [] + for spec in self.dependencies: + found = False + for storage_dir in self.env.GetLibSourceDirs(): + if found: + break + lm = LibraryPackageManager(storage_dir) + pkg = lm.get_package(spec) + if not pkg: + continue + for lb in self.env.GetLibBuilders(): + if pkg.path != lb.path: + continue + if lb not in self.depbuilders: + self.depend_on(lb, recursive=False) + found_lbs.append(lb) + found = True + break + if found: + continue + + # look for built-in libraries by a name + # which don't have package manifest + for lb in self.env.GetLibBuilders(): + if lb.name != spec: + continue + if lb not in self.depbuilders: + self.depend_on(lb) + found = True + break + + # process library dependencies + for lb in found_lbs: + lb.search_deps_recursive() + + def build(self): + self.is_built = True # do not build Project now + result = super().build() + self.env.PrependUnique(CPPPATH=self.get_include_dirs()) + return result + + +def GetLibSourceDirs(env): + items = env.GetProjectOption("lib_extra_dirs", []) + items.extend(env["LIBSOURCE_DIRS"]) + return [ + env.subst(fs.expanduser(item) if item.startswith("~") else item) + for item in items + ] + + +def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): + compat_mode = lb.lib_compat_mode + if lb.name in env.GetProjectOption("lib_ignore", []): + if verbose: + sys.stderr.write("Ignored library %s\n" % lb.path) + return None + if compat_mode == "strict" and not lb.is_platforms_compatible(env["PIOPLATFORM"]): + if verbose: + sys.stderr.write("Platform incompatible library %s\n" % lb.path) + return False + if compat_mode in ("soft", "strict") and not lb.is_frameworks_compatible( + env.get("PIOFRAMEWORK", "__noframework__") + ): + if verbose: + sys.stderr.write("Framework incompatible library %s\n" % lb.path) + return False + return True + + +def GetLibBuilders(_): # pylint: disable=too-many-branches + env = DefaultEnvironment() + if env.get("__PIO_LIB_BUILDERS", None) is not None: + return sorted( + env["__PIO_LIB_BUILDERS"], + key=lambda lb: 0 if lb.is_dependent else 1, + ) + + env.Replace(__PIO_LIB_BUILDERS=[]) + + verbose = int(ARGUMENTS.get("PIOVERBOSE", 0)) + found_incompat = False + + for storage_dir in env.GetLibSourceDirs(): + storage_dir = os.path.abspath(storage_dir) + if not os.path.isdir(storage_dir): + continue + for item in sorted(os.listdir(storage_dir)): + lib_dir = os.path.join(storage_dir, item) + if item == "__cores__": + continue + if LibraryPackageManager.is_symlink(lib_dir): + lib_dir, _ = LibraryPackageManager.resolve_symlink(lib_dir) + if not lib_dir or not os.path.isdir(lib_dir): + continue + try: + lb = LibBuilderFactory.new(env, lib_dir) + except exception.InvalidJSONFile: + if verbose: + sys.stderr.write( + "Skip library with broken manifest: %s\n" % lib_dir + ) + continue + if env.IsCompatibleLibBuilder(lb): + env.Append(__PIO_LIB_BUILDERS=[lb]) + else: + found_incompat = True + + for lb in env.get("EXTRA_LIB_BUILDERS", []): + if env.IsCompatibleLibBuilder(lb): + env.Append(__PIO_LIB_BUILDERS=[lb]) + else: + found_incompat = True + + if verbose and found_incompat: + sys.stderr.write( + 'More details about "Library Compatibility Mode": ' + "https://docs.platformio.org/page/librarymanager/ldf.html#" + "ldf-compat-mode\n" + ) + + return env["__PIO_LIB_BUILDERS"] + + +def ConfigureProjectLibBuilder(env): + _pm_storage = {} + + def _get_lib_license(pkg): + storage_dir = os.path.dirname(os.path.dirname(pkg.path)) + if storage_dir not in _pm_storage: + _pm_storage[storage_dir] = LibraryPackageManager(storage_dir) + try: + return (_pm_storage[storage_dir].load_manifest(pkg) or {}).get("license") + except MissingPackageManifestError: + pass + return None + + def _correct_found_libs(lib_builders): + # build full dependency graph + found_lbs = [lb for lb in lib_builders if lb.is_dependent] + for lb in lib_builders: + if lb in found_lbs: + lb.search_deps_recursive(lb.get_search_files()) + for lb in lib_builders: + for deplb in lb.depbuilders[:]: + if deplb not in found_lbs: + lb.depbuilders.remove(deplb) + + def _print_deps_tree(root, level=0): + margin = "| " * (level) + for lb in root.depbuilders: + title = lb.name + pkg = PackageItem(lb.path) + if pkg.metadata: + title += " @ %s" % pkg.metadata.version + elif lb.version: + title += " @ %s" % lb.version + click.echo("%s|-- %s" % (margin, title), nl=False) + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + click.echo( + " (License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False + ) + if pkg.metadata and pkg.metadata.spec.external: + click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False) + click.echo("Path: %s" % lb.path, nl=False) + click.echo(")", nl=False) + click.echo("") + if lb.verbose and lb.depbuilders: + _print_deps_tree(lb, level + 1) + + project = ProjectAsLibBuilder(env, "$PROJECT_DIR") + + if "test" in env["BUILD_TYPE"]: + project.env.ConfigureTestTarget() + + ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member + + click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf") + click.echo( + "LDF Modes: Finder ~ %s, Compatibility ~ %s" + % (ldf_mode, project.lib_compat_mode) + ) + + project.install_dependencies() + + lib_builders = env.GetLibBuilders() + click.echo("Found %d compatible libraries" % len(lib_builders)) + + click.echo("Scanning dependencies...") + project.search_deps_recursive() + + if ldf_mode.startswith("chain") and project.depbuilders: + _correct_found_libs(lib_builders) + + if project.depbuilders: + click.echo("Dependency Graph") + _print_deps_tree(project) + else: + click.echo("No dependencies") + + return project + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(GetLibSourceDirs) + env.AddMethod(IsCompatibleLibBuilder) + env.AddMethod(GetLibBuilders) + env.AddMethod(ConfigureProjectLibBuilder) + return env diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py new file mode 100644 index 0000000..d20879e --- /dev/null +++ b/platformio/builder/tools/piomaxlen.py @@ -0,0 +1,96 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import re + +from SCons.Platform import TempFileMunge # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Subst import quote_spaces # pylint: disable=import-error + +from platformio.compat import IS_WINDOWS, hashlib_encode_data + +# There are the next limits depending on a platform: +# - Windows = 8192 +# - Unix = 131072 +# We need ~512 characters for compiler and temporary file paths +MAX_LINE_LENGTH = (8192 if IS_WINDOWS else 131072) - 512 + +WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)") + + +def tempfile_arg_esc_func(arg): + arg = quote_spaces(arg) + if not IS_WINDOWS: + return arg + # GCC requires double Windows slashes, let's use UNIX separator + return WINPATHSEP_RE.sub(r"/\1", arg) + + +def long_sources_hook(env, sources): + _sources = str(sources).replace("\\", "/") + if len(str(_sources)) < MAX_LINE_LENGTH: + return sources + + # fix space in paths + data = [] + for line in _sources.split(".o "): + line = line.strip() + if not line.endswith(".o"): + line += ".o" + data.append('"%s"' % line) + + return '@"%s"' % _file_long_data(env, " ".join(data)) + + +def _file_long_data(env, data): + build_dir = env.subst("$BUILD_DIR") + if not os.path.isdir(build_dir): + os.makedirs(build_dir) + tmp_file = os.path.join( + build_dir, "longcmd-%s" % hashlib.md5(hashlib_encode_data(data)).hexdigest() + ) + if os.path.isfile(tmp_file): + return tmp_file + with open(tmp_file, mode="w", encoding="utf8") as fp: + fp.write(data) + return tmp_file + + +def exists(env): + return "compiledb" not in COMMAND_LINE_TARGETS and not env.IsIntegrationDump() + + +def generate(env): + if not exists(env): + return env + kwargs = dict( + _long_sources_hook=long_sources_hook, + TEMPFILE=TempFileMunge, + MAXLINELENGTH=MAX_LINE_LENGTH, + TEMPFILEARGESCFUNC=tempfile_arg_esc_func, + TEMPFILESUFFIX=".tmp", + TEMPFILEDIR="$BUILD_DIR", + ) + + for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"): + kwargs[name] = "${TEMPFILE('%s','$%sSTR')}" % (env.get(name), name) + + kwargs["ARCOM"] = env.get("ARCOM", "").replace( + "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" + ) + env.Replace(**kwargs) + + return env diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py new file mode 100644 index 0000000..47496fe --- /dev/null +++ b/platformio/builder/tools/piomisc.py @@ -0,0 +1,149 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from platformio import fs, util +from platformio.proc import exec_command + + +@util.memoized() +def GetCompilerType(env): + if env.subst("$CC").endswith("-gcc"): + return "gcc" + try: + sysenv = os.environ.copy() + sysenv["PATH"] = str(env["ENV"]["PATH"]) + result = exec_command([env.subst("$CC"), "-v"], env=sysenv) + except OSError: + return None + if result["returncode"] != 0: + return None + output = "".join([result["out"], result["err"]]).lower() + if "clang" in output and "LLVM" in output: + return "clang" + if "gcc" in output: + return "gcc" + return None + + +def GetActualLDScript(env): + def _lookup_in_ldpath(script): + for d in env.get("LIBPATH", []): + path = os.path.join(env.subst(d), script) + if os.path.isfile(path): + return path + return None + + script = None + script_in_next = False + for f in env.get("LINKFLAGS", []): + raw_script = None + if f == "-T": + script_in_next = True + continue + if script_in_next: + script_in_next = False + raw_script = f + elif f.startswith("-Wl,-T"): + raw_script = f[6:] + else: + continue + script = env.subst(raw_script.replace('"', "").strip()) + if os.path.isfile(script): + return script + path = _lookup_in_ldpath(script) + if path: + return path + + if script: + sys.stderr.write( + "Error: Could not find '%s' LD script in LDPATH '%s'\n" + % (script, env.subst("$LIBPATH")) + ) + env.Exit(1) + + if not script and "LDSCRIPT_PATH" in env: + path = _lookup_in_ldpath(env["LDSCRIPT_PATH"]) + if path: + return path + + sys.stderr.write("Error: Could not find LD script\n") + env.Exit(1) + + +def ConfigureDebugTarget(env): + def _cleanup_debug_flags(scope): + if scope not in env: + return + unflags = ["-Os", "-g"] + for level in [0, 1, 2, 3]: + for flag in ("O", "g", "ggdb"): + unflags.append("-%s%d" % (flag, level)) + env[scope] = [f for f in env.get(scope, []) if f not in unflags] + + env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) + + for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): + _cleanup_debug_flags(scope) + + debug_flags = env.ParseFlags( + env.get("PIODEBUGFLAGS") + if env.get("PIODEBUGFLAGS") + and not env.GetProjectOptions(as_dict=True).get("debug_build_flags") + else env.GetProjectOption("debug_build_flags") + ) + + env.MergeFlags(debug_flags) + optimization_flags = [ + f for f in debug_flags.get("CCFLAGS", []) if f.startswith(("-O", "-g")) + ] + + if optimization_flags: + env.AppendUnique( + ASFLAGS=[ + # skip -O flags for assembler + f + for f in optimization_flags + if f.startswith("-g") + ], + LINKFLAGS=optimization_flags, + ) + + +def GetExtraScripts(env, scope): + items = [] + for item in env.GetProjectOption("extra_scripts", []): + if scope == "post" and ":" not in item: + items.append(item) + elif item.startswith("%s:" % scope): + items.append(item[len(scope) + 1 :]) + if not items: + return items + with fs.cd(env.subst("$PROJECT_DIR")): + return [os.path.abspath(env.subst(item)) for item in items] + + +def generate(env): + env.AddMethod(GetCompilerType) + env.AddMethod(GetActualLDScript) + env.AddMethod(ConfigureDebugTarget) + env.AddMethod(GetExtraScripts) + # bakward-compatibility with Zephyr build script + env.AddMethod(ConfigureDebugTarget, "ConfigureDebugFlags") + + +def exists(_): + return True diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py new file mode 100644 index 0000000..4ce4b91 --- /dev/null +++ b/platformio/builder/tools/pioplatform.py @@ -0,0 +1,248 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error + +from platformio import fs, util +from platformio.compat import IS_MACOS, IS_WINDOWS +from platformio.package.meta import PackageItem +from platformio.package.version import get_original_version +from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectOptions + +# pylint: disable=too-many-branches, too-many-locals + + +@util.memoized() +def _PioPlatform(): + env = DefaultEnvironment() + return PlatformFactory.from_env(env["PIOENV"], targets=COMMAND_LINE_TARGETS) + + +def PioPlatform(_): + return _PioPlatform() + + +def BoardConfig(env, board=None): + with fs.cd(env.subst("$PROJECT_DIR")): + try: + p = env.PioPlatform() + board = board or env.get("BOARD") + assert board, "BoardConfig: Board is not defined" + return p.board_config(board) + except (AssertionError, UnknownBoard) as exc: + sys.stderr.write("Error: %s\n" % str(exc)) + env.Exit(1) + return None + + +def GetFrameworkScript(env, framework): + p = env.PioPlatform() + assert p.frameworks and framework in p.frameworks + script_path = env.subst(p.frameworks[framework]["script"]) + if not os.path.isfile(script_path): + script_path = os.path.join(p.get_dir(), script_path) + return script_path + + +def LoadPioPlatform(env): + p = env.PioPlatform() + + # Ensure real platform name + env["PIOPLATFORM"] = p.name + + # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH + for pkg in p.get_installed_packages(): + type_ = p.get_package_type(pkg.metadata.name) + if type_ not in ("toolchain", "uploader", "debugger"): + continue + env.PrependENVPath( + "PATH", + os.path.join(pkg.path, "bin") + if os.path.isdir(os.path.join(pkg.path, "bin")) + else pkg.path, + ) + if ( + not IS_WINDOWS + and os.path.isdir(os.path.join(pkg.path, "lib")) + and type_ != "toolchain" + ): + env.PrependENVPath( + "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH", + os.path.join(pkg.path, "lib"), + ) + + # Platform specific LD Scripts + if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")): + env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")]) + + if "BOARD" not in env: + return + + # update board manifest with overridden data from INI config + board_config = env.BoardConfig() + for option, value in env.GetProjectOptions(): + if not option.startswith("board_"): + continue + option = option.lower()[6:] + try: + if isinstance(board_config.get(option), bool): + value = str(value).lower() in ("1", "yes", "true") + elif isinstance(board_config.get(option), int): + value = int(value) + except KeyError: + pass + board_config.update(option, value) + + # load default variables from board config + for option_meta in ProjectOptions.values(): + if not option_meta.buildenvvar or option_meta.buildenvvar in env: + continue + data_path = ( + option_meta.name[6:] + if option_meta.name.startswith("board_") + else option_meta.name.replace("_", ".") + ) + try: + env[option_meta.buildenvvar] = board_config.get(data_path) + except KeyError: + pass + + if "build.ldscript" in board_config: + env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript")) + + +def PrintConfiguration(env): # pylint: disable=too-many-statements + platform = env.PioPlatform() + pkg_metadata = PackageItem(platform.get_dir()).metadata + board_config = env.BoardConfig() if "BOARD" in env else None + + def _get_configuration_data(): + return ( + None + if not board_config + else [ + "CONFIGURATION:", + "https://docs.platformio.org/page/boards/%s/%s.html" + % (platform.name, board_config.id), + ] + ) + + def _get_plaform_data(): + data = [ + "PLATFORM: %s (%s)" + % ( + platform.title, + pkg_metadata.version if pkg_metadata else platform.version, + ) + ] + if ( + int(ARGUMENTS.get("PIOVERBOSE", 0)) + and pkg_metadata + and pkg_metadata.spec.external + ): + data.append("(%s)" % pkg_metadata.spec.uri) + if board_config: + data.extend([">", board_config.get("name")]) + return data + + def _get_hardware_data(): + data = ["HARDWARE:"] + mcu = env.subst("$BOARD_MCU") + f_cpu = env.subst("$BOARD_F_CPU") + if mcu: + data.append(mcu.upper()) + if f_cpu: + f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()])) + data.append("%dMHz," % (f_cpu / 1000000)) + if not board_config: + return data + ram = board_config.get("upload", {}).get("maximum_ram_size") + flash = board_config.get("upload", {}).get("maximum_size") + data.append( + "%s RAM, %s Flash" + % (fs.humanize_file_size(ram), fs.humanize_file_size(flash)) + ) + return data + + def _get_debug_data(): + debug_tools = ( + board_config.get("debug", {}).get("tools") if board_config else None + ) + if not debug_tools: + return None + data = [ + "DEBUG:", + "Current", + "(%s)" + % board_config.get_debug_tool_name(env.GetProjectOption("debug_tool")), + ] + onboard = [] + external = [] + for key, value in debug_tools.items(): + if value.get("onboard"): + onboard.append(key) + else: + external.append(key) + if onboard: + data.extend(["On-board", "(%s)" % ", ".join(sorted(onboard))]) + if external: + data.extend(["External", "(%s)" % ", ".join(sorted(external))]) + return data + + def _get_packages_data(): + data = [] + for item in platform.dump_used_packages(): + original_version = get_original_version(item["version"]) + info = "%s @ %s" % (item["name"], item["version"]) + extra = [] + if original_version: + extra.append(original_version) + if "src_url" in item and int(ARGUMENTS.get("PIOVERBOSE", 0)): + extra.append(item["src_url"]) + if extra: + info += " (%s)" % ", ".join(extra) + data.append(info) + if not data: + return None + return ["PACKAGES:"] + ["\n - %s" % d for d in sorted(data)] + + for data in ( + _get_configuration_data(), + _get_plaform_data(), + _get_hardware_data(), + _get_debug_data(), + _get_packages_data(), + ): + if data and len(data) > 1: + print(" ".join(data)) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(PioPlatform) + env.AddMethod(BoardConfig) + env.AddMethod(GetFrameworkScript) + env.AddMethod(LoadPioPlatform) + env.AddMethod(PrintConfiguration) + return env diff --git a/platformio/builder/tools/pioproject.py b/platformio/builder/tools/pioproject.py new file mode 100644 index 0000000..9b7b149 --- /dev/null +++ b/platformio/builder/tools/pioproject.py @@ -0,0 +1,54 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.compat import MISSING +from platformio.project.config import ProjectConfig + + +def GetProjectConfig(env): + return ProjectConfig.get_instance(env["PROJECT_CONFIG"]) + + +def GetProjectOptions(env, as_dict=False): + return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict) + + +def GetProjectOption(env, option, default=MISSING): + return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default) + + +def LoadProjectOptions(env): + config = env.GetProjectConfig() + section = "env:" + env["PIOENV"] + for option in config.options(section): + option_meta = config.find_option_meta(section, option) + if ( + not option_meta + or not option_meta.buildenvvar + or option_meta.buildenvvar in env + ): + continue + env[option_meta.buildenvvar] = config.get(section, option) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(GetProjectConfig) + env.AddMethod(GetProjectOptions) + env.AddMethod(GetProjectOption) + env.AddMethod(LoadProjectOptions) + return env diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py new file mode 100644 index 0000000..3ac2431 --- /dev/null +++ b/platformio/builder/tools/piosize.py @@ -0,0 +1,266 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-locals + +import json +import sys +from os import environ, makedirs, remove +from os.path import isdir, join, splitdrive + +from elftools.elf.descriptions import describe_sh_flags +from elftools.elf.elffile import ELFFile + +from platformio.compat import IS_WINDOWS +from platformio.proc import exec_command + + +def _run_tool(cmd, env, tool_args): + sysenv = environ.copy() + sysenv["PATH"] = str(env["ENV"]["PATH"]) + + build_dir = env.subst("$BUILD_DIR") + if not isdir(build_dir): + makedirs(build_dir) + tmp_file = join(build_dir, "size-data-longcmd.txt") + + with open(tmp_file, mode="w", encoding="utf8") as fp: + fp.write("\n".join(tool_args)) + + cmd.append("@" + tmp_file) + result = exec_command(cmd, env=sysenv) + remove(tmp_file) + + return result + + +def _get_symbol_locations(env, elf_path, addrs): + if not addrs: + return {} + cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path] + result = _run_tool(cmd, env, addrs) + locations = [line for line in result["out"].split("\n") if line] + assert len(addrs) == len(locations) + + return dict(zip(addrs, [loc.strip() for loc in locations])) + + +def _get_demangled_names(env, mangled_names): + if not mangled_names: + return {} + result = _run_tool( + [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names + ) + demangled_names = [line for line in result["out"].split("\n") if line] + assert len(mangled_names) == len(demangled_names) + + return dict( + zip( + mangled_names, + [dn.strip().replace("::__FUNCTION__", "") for dn in demangled_names], + ) + ) + + +def _collect_sections_info(env, elffile): + sections = {} + for section in elffile.iter_sections(): + if section.is_null() or section.name.startswith(".debug"): + continue + + section_type = section["sh_type"] + section_flags = describe_sh_flags(section["sh_flags"]) + section_size = section.data_size + + section_data = { + "name": section.name, + "size": section_size, + "start_addr": section["sh_addr"], + "type": section_type, + "flags": section_flags, + } + + sections[section.name] = section_data + sections[section.name]["in_flash"] = env.pioSizeIsFlashSection(section_data) + sections[section.name]["in_ram"] = env.pioSizeIsRamSection(section_data) + + return sections + + +def _collect_symbols_info(env, elffile, elf_path, sections): + symbols = [] + + symbol_section = elffile.get_section_by_name(".symtab") + if symbol_section.is_null(): + sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?") + env.Exit(1) + + sysenv = environ.copy() + sysenv["PATH"] = str(env["ENV"]["PATH"]) + + symbol_addrs = [] + mangled_names = [] + for s in symbol_section.iter_symbols(): + symbol_info = s.entry["st_info"] + symbol_addr = s["st_value"] + symbol_size = s["st_size"] + symbol_type = symbol_info["type"] + + if not env.pioSizeIsValidSymbol(s.name, symbol_type, symbol_addr): + continue + + symbol = { + "addr": symbol_addr, + "bind": symbol_info["bind"], + "name": s.name, + "type": symbol_type, + "size": symbol_size, + "section": env.pioSizeDetermineSection(sections, symbol_addr), + } + + if s.name.startswith("_Z"): + mangled_names.append(s.name) + + symbol_addrs.append(hex(symbol_addr)) + symbols.append(symbol) + + symbol_locations = _get_symbol_locations(env, elf_path, symbol_addrs) + demangled_names = _get_demangled_names(env, mangled_names) + for symbol in symbols: + if symbol["name"].startswith("_Z"): + symbol["demangled_name"] = demangled_names.get(symbol["name"]) + location = symbol_locations.get(hex(symbol["addr"])) + if not location or "?" in location: + continue + if IS_WINDOWS: + drive, tail = splitdrive(location) + location = join(drive.upper(), tail) + symbol["file"] = location + symbol["line"] = 0 + if ":" in location: + file_, line = location.rsplit(":", 1) + if line.isdigit(): + symbol["file"] = file_ + symbol["line"] = int(line) + return symbols + + +def pioSizeDetermineSection(_, sections, symbol_addr): + for section, info in sections.items(): + if not info.get("in_flash", False) and not info.get("in_ram", False): + continue + if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]): + return section + return "unknown" + + +def pioSizeIsValidSymbol(_, symbol_name, symbol_type, symbol_address): + return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE" + + +def pioSizeIsRamSection(_, section): + return ( + section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS") + and section.get("flags", "") == "WA" + ) + + +def pioSizeIsFlashSection(_, section): + return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "") + + +def pioSizeCalculateFirmwareSize(_, sections): + flash_size = ram_size = 0 + for section_info in sections.values(): + if section_info.get("in_flash", False): + flash_size += section_info.get("size", 0) + if section_info.get("in_ram", False): + ram_size += section_info.get("size", 0) + + return ram_size, flash_size + + +def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument + data = {"device": {}, "memory": {}, "version": 1} + + board = env.BoardConfig() + if board: + data["device"] = { + "mcu": board.get("build.mcu", ""), + "cpu": board.get("build.cpu", ""), + "frequency": board.get("build.f_cpu"), + "flash": int(board.get("upload.maximum_size", 0)), + "ram": int(board.get("upload.maximum_ram_size", 0)), + } + if data["device"]["frequency"] and data["device"]["frequency"].endswith("L"): + data["device"]["frequency"] = int(data["device"]["frequency"][0:-1]) + + elf_path = env.subst("$PIOMAINPROG") + + with open(elf_path, "rb") as fp: + elffile = ELFFile(fp) + + if not elffile.has_dwarf_info(): + sys.stderr.write("Elf file doesn't contain DWARF information") + env.Exit(1) + + sections = _collect_sections_info(env, elffile) + firmware_ram, firmware_flash = env.pioSizeCalculateFirmwareSize(sections) + data["memory"]["total"] = { + "ram_size": firmware_ram, + "flash_size": firmware_flash, + "sections": sections, + } + + files = {} + for symbol in _collect_symbols_info(env, elffile, elf_path, sections): + file_path = symbol.get("file") or "unknown" + if not files.get(file_path, {}): + files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0} + + symbol_size = symbol.get("size", 0) + section = sections.get(symbol.get("section", ""), {}) + if not section: + continue + if section.get("in_ram", False): + files[file_path]["ram_size"] += symbol_size + if section.get("in_flash", False): + files[file_path]["flash_size"] += symbol_size + + files[file_path]["symbols"].append(symbol) + + data["memory"]["files"] = [] + for k, v in files.items(): + file_data = {"path": k} + file_data.update(v) + data["memory"]["files"].append(file_data) + + with open( + join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8" + ) as fp: + fp.write(json.dumps(data)) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(pioSizeIsRamSection) + env.AddMethod(pioSizeIsFlashSection) + env.AddMethod(pioSizeCalculateFirmwareSize) + env.AddMethod(pioSizeDetermineSection) + env.AddMethod(pioSizeIsValidSymbol) + env.AddMethod(DumpSizeData) + return env diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py new file mode 100644 index 0000000..32bb5ff --- /dev/null +++ b/platformio/builder/tools/piotarget.py @@ -0,0 +1,116 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from SCons.Action import Action # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error + +from platformio import compat, fs + + +def VerboseAction(_, act, actstr): + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + return act + return Action(act, actstr) + + +def IsCleanTarget(env): + return env.GetOption("clean") + + +def CleanProject(env, fullclean=False): + def _relpath(path): + if compat.IS_WINDOWS: + prefix = os.getcwd()[:2].lower() + if ( + ":" not in prefix + or not path.lower().startswith(prefix) + or os.path.relpath(path).startswith("..") + ): + return path + return os.path.relpath(path) + + def _clean_dir(path): + clean_rel_path = _relpath(path) + print(f"Removing {clean_rel_path}") + fs.rmtree(path) + + build_dir = env.subst("$BUILD_DIR") + libdeps_dir = env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) + if os.path.isdir(build_dir): + _clean_dir(build_dir) + else: + print("Build environment is clean") + + if fullclean and os.path.isdir(libdeps_dir): + _clean_dir(libdeps_dir) + + print("Done cleaning") + + +def AddTarget( # pylint: disable=too-many-arguments + env, + name, + dependencies, + actions, + title=None, + description=None, + group="General", + always_build=True, +): + if "__PIO_TARGETS" not in env: + env["__PIO_TARGETS"] = {} + assert name not in env["__PIO_TARGETS"] + env["__PIO_TARGETS"][name] = dict( + name=name, title=title, description=description, group=group + ) + target = env.Alias(name, dependencies, actions) + if always_build: + AlwaysBuild(target) + return target + + +def AddPlatformTarget(env, *args, **kwargs): + return env.AddTarget(group="Platform", *args, **kwargs) + + +def AddCustomTarget(env, *args, **kwargs): + return env.AddTarget(group="Custom", *args, **kwargs) + + +def DumpTargets(env): + targets = env.get("__PIO_TARGETS") or {} + # pre-fill default targets if embedded dev-platform + if env.PioPlatform().is_embedded() and not any( + t["group"] == "Platform" for t in targets.values() + ): + targets["upload"] = dict(name="upload", group="Platform", title="Upload") + return list(targets.values()) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(VerboseAction) + env.AddMethod(IsCleanTarget) + env.AddMethod(CleanProject) + env.AddMethod(AddTarget) + env.AddMethod(AddPlatformTarget) + env.AddMethod(AddCustomTarget) + env.AddMethod(DumpTargets) + return env diff --git a/platformio/builder/tools/piotest.py b/platformio/builder/tools/piotest.py new file mode 100644 index 0000000..b3b324b --- /dev/null +++ b/platformio/builder/tools/piotest.py @@ -0,0 +1,61 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio.builder.tools import piobuild +from platformio.test.result import TestSuite +from platformio.test.runners.factory import TestRunnerFactory + + +def ConfigureTestTarget(env): + env.Append( + CPPDEFINES=["UNIT_TEST"], # deprecated, use PIO_UNIT_TESTING + PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piobuild.SRC_BUILD_EXT], + ) + env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"]) + + if "PIOTEST_RUNNING_NAME" in env: + test_name = env["PIOTEST_RUNNING_NAME"] + while True: + test_name = os.path.dirname(test_name) # parent dir + # skip nested tests (user's side issue?) + if not test_name or os.path.basename(test_name).startswith("test_"): + break + env.Prepend( + PIOTEST_SRC_FILTER=[ + f"+<{test_name}{os.path.sep}*.{ext}>" + for ext in piobuild.SRC_BUILD_EXT + ], + CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)], + ) + + env.Prepend( + PIOTEST_SRC_FILTER=[f"+<$PIOTEST_RUNNING_NAME{os.path.sep}>"], + CPPPATH=[os.path.join("$PROJECT_TEST_DIR", "$PIOTEST_RUNNING_NAME")], + ) + + test_runner = TestRunnerFactory.new( + TestSuite(env["PIOENV"], env.get("PIOTEST_RUNNING_NAME", "*")), + env.GetProjectConfig(), + ) + test_runner.configure_build_env(env) + + +def generate(env): + env.AddMethod(ConfigureTestTarget) + + +def exists(_): + return True diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py new file mode 100644 index 0000000..77bf28d --- /dev/null +++ b/platformio/builder/tools/pioupload.py @@ -0,0 +1,258 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import os +import re +import sys +from shutil import copyfile +from time import sleep + +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from serial import Serial, SerialException + +from platformio import exception, fs +from platformio.device.finder import SerialPortFinder, find_mbed_disk, is_pattern_port +from platformio.device.list.util import list_serial_ports +from platformio.proc import exec_command + + +def FlushSerialBuffer(env, port): + s = Serial(env.subst(port)) + s.flushInput() + s.setDTR(False) + s.setRTS(False) + sleep(0.1) + s.setDTR(True) + s.setRTS(True) + s.close() + + +def TouchSerialPort(env, port, baudrate): + port = env.subst(port) + print("Forcing reset using %dbps open/close on port %s" % (baudrate, port)) + try: + s = Serial(port=port, baudrate=baudrate) + s.setDTR(False) + s.close() + except: # pylint: disable=bare-except + pass + sleep(0.4) # DO NOT REMOVE THAT (required by SAM-BA based boards) + + +def WaitForNewSerialPort(env, before): + print("Waiting for the new upload port...") + prev_port = env.subst("$UPLOAD_PORT") + new_port = None + elapsed = 0 + before = [p["port"] for p in before] + while elapsed < 5 and new_port is None: + now = [p["port"] for p in list_serial_ports()] + for p in now: + if p not in before: + new_port = p + break + before = now + sleep(0.25) + elapsed += 0.25 + + if not new_port: + for p in now: + if prev_port == p: + new_port = p + break + + try: + s = Serial(new_port) + s.close() + except SerialException: + sleep(1) + + if not new_port: + sys.stderr.write( + "Error: Couldn't find a board on the selected port. " + "Check that you have the correct port selected. " + "If it is correct, try pressing the board's reset " + "button after initiating the upload.\n" + ) + env.Exit(1) + + return new_port + + +def AutodetectUploadPort(*args, **kwargs): + env = args[0] + initial_port = env.subst("$UPLOAD_PORT") + upload_protocol = env.subst("$UPLOAD_PROTOCOL") + if initial_port and not is_pattern_port(initial_port): + print(env.subst("Using manually specified: $UPLOAD_PORT")) + return + + if upload_protocol == "mbed" or ( + "mbed" in env.subst("$PIOFRAMEWORK") and not upload_protocol + ): + env.Replace(UPLOAD_PORT=find_mbed_disk(initial_port)) + else: + try: + fs.ensure_udev_rules() + except exception.InvalidUdevRules as exc: + sys.stderr.write("\n%s\n\n" % exc) + env.Replace( + UPLOAD_PORT=SerialPortFinder( + board_config=env.BoardConfig() if "BOARD" in env else None, + upload_protocol=upload_protocol, + prefer_gdb_port="blackmagic" in upload_protocol, + verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)), + ).find(initial_port) + ) + + if env.subst("$UPLOAD_PORT"): + print(env.subst("Auto-detected: $UPLOAD_PORT")) + else: + sys.stderr.write( + "Error: Please specify `upload_port` for environment or use " + "global `--upload-port` option.\n" + "For some development platforms it can be a USB flash " + "drive (i.e. /media//)\n" + ) + env.Exit(1) + + +def UploadToDisk(_, target, source, env): + assert "UPLOAD_PORT" in env + progname = env.subst("$PROGNAME") + for ext in ("bin", "hex"): + fpath = os.path.join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext)) + if not os.path.isfile(fpath): + continue + copyfile( + fpath, os.path.join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)) + ) + print( + "Firmware has been successfully uploaded.\n" + "(Some boards may require manual hard reset)" + ) + + +def CheckUploadSize(_, target, source, env): + check_conditions = [ + env.get("BOARD"), + env.get("SIZETOOL") or env.get("SIZECHECKCMD"), + ] + if not all(check_conditions): + return + program_max_size = int(env.BoardConfig().get("upload.maximum_size", 0)) + data_max_size = int(env.BoardConfig().get("upload.maximum_ram_size", 0)) + if program_max_size == 0: + return + + def _configure_defaults(): + env.Replace( + SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", + SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", + SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+", + ) + + def _get_size_output(): + cmd = env.get("SIZECHECKCMD") + if not cmd: + return None + if not isinstance(cmd, list): + cmd = cmd.split() + cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg] + sysenv = os.environ.copy() + sysenv["PATH"] = str(env["ENV"]["PATH"]) + result = exec_command(env.subst(cmd), env=sysenv) + if result["returncode"] != 0: + return None + return result["out"].strip() + + def _calculate_size(output, pattern): + if not output or not pattern: + return -1 + size = 0 + regexp = re.compile(pattern) + for line in output.split("\n"): + line = line.strip() + if not line: + continue + match = regexp.search(line) + if not match: + continue + size += sum(int(value) for value in match.groups()) + return size + + def _format_availale_bytes(value, total): + percent_raw = float(value) / float(total) + blocks_per_progress = 10 + used_blocks = min( + int(round(blocks_per_progress * percent_raw)), blocks_per_progress + ) + return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format( + "=" * used_blocks, blocks_per_progress, percent_raw, value, total + ) + + if not env.get("SIZECHECKCMD") and not env.get("SIZEPROGREGEXP"): + _configure_defaults() + output = _get_size_output() + program_size = _calculate_size(output, env.get("SIZEPROGREGEXP")) + data_size = _calculate_size(output, env.get("SIZEDATAREGEXP")) + + print('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"') + if data_max_size and data_size > -1: + print("RAM: %s" % _format_availale_bytes(data_size, data_max_size)) + if program_size > -1: + print("Flash: %s" % _format_availale_bytes(program_size, program_max_size)) + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + print(output) + + # raise error + # if data_max_size and data_size > data_max_size: + # sys.stderr.write( + # "Error: The data size (%d bytes) is greater " + # "than maximum allowed (%s bytes)\n" % (data_size, data_max_size)) + # env.Exit(1) + if program_size > program_max_size: + sys.stderr.write( + "Error: The program size (%d bytes) is greater " + "than maximum allowed (%s bytes)\n" % (program_size, program_max_size) + ) + env.Exit(1) + + +def PrintUploadInfo(env): + configured = env.subst("$UPLOAD_PROTOCOL") + available = [configured] if configured else [] + if "BOARD" in env: + available.extend(env.BoardConfig().get("upload", {}).get("protocols", [])) + if available: + print("AVAILABLE: %s" % ", ".join(sorted(set(available)))) + if configured: + print("CURRENT: upload_protocol = %s" % configured) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(FlushSerialBuffer) + env.AddMethod(TouchSerialPort) + env.AddMethod(WaitForNewSerialPort) + env.AddMethod(AutodetectUploadPort) + env.AddMethod(UploadToDisk) + env.AddMethod(CheckUploadSize) + env.AddMethod(PrintUploadInfo) + return env diff --git a/platformio/cache.py b/platformio/cache.py new file mode 100644 index 0000000..2090154 --- /dev/null +++ b/platformio/cache.py @@ -0,0 +1,165 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import codecs +import hashlib +import os +from time import time + +from platformio import app, fs +from platformio.compat import hashlib_encode_data +from platformio.package.lockfile import LockFile +from platformio.project.helpers import get_project_cache_dir + + +class ContentCache: + def __init__(self, namespace=None): + self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content") + self._db_path = os.path.join(self.cache_dir, "db.data") + self._lockfile = None + if not os.path.isdir(self.cache_dir): + os.makedirs(self.cache_dir) + + def __enter__(self): + # cleanup obsolete items + self.delete() + return self + + def __exit__(self, type_, value, traceback): + pass + + @staticmethod + def key_from_args(*args): + h = hashlib.sha1() + for arg in args: + if arg: + h.update(hashlib_encode_data(arg)) + return h.hexdigest() + + def get_cache_path(self, key): + assert "/" not in key and "\\" not in key + key = str(key) + assert len(key) > 3 + return os.path.join(self.cache_dir, key) + + def get(self, key): + cache_path = self.get_cache_path(key) + if not os.path.isfile(cache_path): + return None + with codecs.open(cache_path, "rb", encoding="utf8") as fp: + return fp.read() + + def set(self, key, data, valid): + if not app.get_setting("enable_cache"): + return False + cache_path = self.get_cache_path(key) + if os.path.isfile(cache_path): + self.delete(key) + if not data: + return False + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert valid.endswith(tuple(tdmap)) + expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) + + if not self._lock_dbindex(): + return False + + if not os.path.isdir(os.path.dirname(cache_path)): + os.makedirs(os.path.dirname(cache_path)) + try: + with codecs.open(cache_path, mode="wb", encoding="utf8") as fp: + fp.write(data) + with open(self._db_path, mode="a", encoding="utf8") as fp: + fp.write("%s=%s\n" % (str(expire_time), os.path.basename(cache_path))) + except UnicodeError: + if os.path.isfile(cache_path): + try: + os.remove(cache_path) + except OSError: + pass + + return self._unlock_dbindex() + + def delete(self, keys=None): + """Keys=None, delete expired items""" + if not os.path.isfile(self._db_path): + return None + if not keys: + keys = [] + if not isinstance(keys, list): + keys = [keys] + paths_for_delete = [self.get_cache_path(k) for k in keys] + found = False + newlines = [] + with open(self._db_path, encoding="utf8") as fp: + for line in fp.readlines(): + line = line.strip() + if "=" not in line: + continue + expire, fname = line.split("=") + path = os.path.join(self.cache_dir, fname) + try: + if ( + time() < int(expire) + and os.path.isfile(path) + and path not in paths_for_delete + ): + newlines.append(line) + continue + except ValueError: + pass + found = True + if os.path.isfile(path): + try: + os.remove(path) + if not os.listdir(os.path.dirname(path)): + fs.rmtree(os.path.dirname(path)) + except OSError: + pass + + if found and self._lock_dbindex(): + with open(self._db_path, mode="w", encoding="utf8") as fp: + fp.write("\n".join(newlines) + "\n") + self._unlock_dbindex() + + return True + + def clean(self): + if not os.path.isdir(self.cache_dir): + return + fs.rmtree(self.cache_dir) + + def _lock_dbindex(self): + self._lockfile = LockFile(self.cache_dir) + try: + self._lockfile.acquire() + except: # pylint: disable=bare-except + return False + + return True + + def _unlock_dbindex(self): + if self._lockfile: + self._lockfile.release() + return True + + +# +# Helpers +# + + +def cleanup_content_cache(namespace=None): + with ContentCache(namespace) as cc: + cc.clean() diff --git a/platformio/check/__init__.py b/platformio/check/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/check/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/check/cli.py b/platformio/check/cli.py new file mode 100644 index 0000000..d99da31 --- /dev/null +++ b/platformio/check/cli.py @@ -0,0 +1,334 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches +# pylint: disable=redefined-builtin,too-many-statements + +import json +import os +import shutil +from collections import Counter +from os.path import dirname, isfile +from time import time + +import click +from tabulate import tabulate + +from platformio import app, exception, fs, util +from platformio.check.defect import DefectItem +from platformio.check.tools import CheckToolFactory +from platformio.project.config import ProjectConfig +from platformio.project.helpers import find_project_dir_above, get_project_dir + + +@click.command("check", short_help="Static Code Analysis") +@click.option("-e", "--environment", multiple=True) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), +) +@click.option( + "-c", + "--project-conf", + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), +) +@click.option("--pattern", multiple=True, hidden=True) +@click.option("-f", "--src-filters", multiple=True) +@click.option("--flags", multiple=True) +@click.option( + "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) +) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.option("--json-output", is_flag=True) +@click.option( + "--fail-on-defect", + multiple=True, + type=click.Choice(DefectItem.SEVERITY_LABELS.values()), +) +@click.option("--skip-packages", is_flag=True) +def cli( + environment, + project_dir, + project_conf, + src_filters, + pattern, + flags, + severity, + silent, + verbose, + json_output, + fail_on_defect, + skip_packages, +): + app.set_session_var("custom_project_conf", project_conf) + + # find project directory on upper level + if isfile(project_dir): + project_dir = find_project_dir_above(project_dir) + + results = [] + with fs.cd(project_dir): + config = ProjectConfig.get_instance(project_conf) + config.validate(environment) + + default_envs = config.default_envs() + for envname in config.envs(): + skipenv = any( + [ + environment and envname not in environment, + not environment and default_envs and envname not in default_envs, + ] + ) + + env_options = config.items(env=envname, as_dict=True) + env_dump = [] + for k, v in env_options.items(): + if k not in ("platform", "framework", "board"): + continue + env_dump.append( + "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) + ) + + default_src_filters = [ + "+<%s>" % os.path.basename(config.get("platformio", "src_dir")), + "+<%s>" % os.path.basename(config.get("platformio", "include_dir")), + ] + + src_filters = ( + src_filters + or pattern + or env_options.get( + "check_src_filters", + env_options.get("check_patterns", default_src_filters), + ) + ) + + tool_options = dict( + verbose=verbose, + silent=silent, + src_filters=src_filters, + flags=flags or env_options.get("check_flags"), + severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] + if silent + else severity or config.get("env:" + envname, "check_severity"), + skip_packages=skip_packages or env_options.get("check_skip_packages"), + platform_packages=env_options.get("platform_packages"), + ) + + for tool in config.get("env:" + envname, "check_tool"): + if skipenv: + results.append({"env": envname, "tool": tool}) + continue + if not silent and not json_output: + print_processing_header(tool, envname, env_dump) + + ct = CheckToolFactory.new( + tool, project_dir, config, envname, tool_options + ) + + result = {"env": envname, "tool": tool, "duration": time()} + rc = ct.check( + on_defect_callback=None + if (json_output or verbose) + else lambda defect: click.echo(repr(defect)) + ) + + result["defects"] = ct.get_defects() + result["duration"] = time() - result["duration"] + + result["succeeded"] = rc == 0 + if fail_on_defect: + result["succeeded"] = rc == 0 and not any( + DefectItem.SEVERITY_LABELS[d.severity] in fail_on_defect + for d in result["defects"] + ) + result["stats"] = collect_component_stats(result) + results.append(result) + + if verbose: + click.echo("\n".join(repr(d) for d in result["defects"])) + + if not json_output and not silent: + if rc != 0: + click.echo( + "Error: %s failed to perform check! Please " + "examine tool output in verbose mode." % tool + ) + elif not result["defects"]: + click.echo("No defects found") + print_processing_footer(result) + + if json_output: + click.echo(json.dumps(results_to_json(results))) + elif not silent: + print_check_summary(results, verbose=verbose) + + # Reset custom project config + app.set_session_var("custom_project_conf", None) + + command_failed = any(r.get("succeeded") is False for r in results) + if command_failed: + raise exception.ReturnErrorCode(1) + + +def results_to_json(raw): + results = [] + for item in raw: + if item.get("succeeded") is None: + continue + item.update( + { + "succeeded": bool(item.get("succeeded")), + "defects": [d.as_dict() for d in item.get("defects", [])], + } + ) + results.append(item) + + return results + + +def print_processing_header(tool, envname, envdump): + click.echo( + "Checking %s > %s (%s)" + % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump)) + ) + terminal_width = shutil.get_terminal_size().columns + click.secho("-" * terminal_width, bold=True) + + +def print_processing_footer(result): + is_failed = not result.get("succeeded") + util.print_labeled_bar( + "[%s] Took %.2f seconds" + % ( + ( + click.style("FAILED", fg="red", bold=True) + if is_failed + else click.style("PASSED", fg="green", bold=True) + ), + result["duration"], + ), + is_error=is_failed, + ) + + +def collect_component_stats(result): + components = {} + + def _append_defect(component, defect): + if not components.get(component): + components[component] = Counter() + components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1}) + + for defect in result.get("defects", []): + component = dirname(defect.file) or defect.file + _append_defect(component, defect) + + if component.lower().startswith(get_project_dir().lower()): + while os.sep in component: + component = dirname(component) + _append_defect(component, defect) + + return components + + +def print_defects_stats(results): + if not results: + return + + component_stats = {} + for r in results: + for k, v in r.get("stats", {}).items(): + if not component_stats.get(k): + component_stats[k] = Counter() + component_stats[k].update(r["stats"][k]) + + if not component_stats: + return + + severity_labels = list(DefectItem.SEVERITY_LABELS.values()) + severity_labels.reverse() + tabular_data = [] + for k, v in component_stats.items(): + tool_defect = [v.get(s, 0) for s in severity_labels] + tabular_data.append([k] + tool_defect) + + total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]] + tabular_data.sort() + tabular_data.append([]) # Empty line as delimiter + tabular_data.append(total) + + headers = ["Component"] + headers.extend([label.upper() for label in severity_labels]) + headers = [click.style(h, bold=True) for h in headers] + click.echo(tabulate(tabular_data, headers=headers, numalign="center")) + click.echo() + + +def print_check_summary(results, verbose=False): + click.echo() + + tabular_data = [] + succeeded_nums = 0 + failed_nums = 0 + duration = 0 + + print_defects_stats(results) + + for result in results: + duration += result.get("duration", 0) + if result.get("succeeded") is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif result.get("succeeded") is None: + status_str = "IGNORED" + if not verbose: + continue + else: + succeeded_nums += 1 + status_str = click.style("PASSED", fg="green") + + tabular_data.append( + ( + click.style(result["env"], fg="cyan"), + result["tool"], + status_str, + util.humanize_duration_time(result.get("duration")), + ) + ) + + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Tool", "Status", "Duration") + ], + ), + err=failed_nums, + ) + + util.print_labeled_bar( + "%s%d succeeded in %s" + % ( + "%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, + util.humanize_duration_time(duration), + ), + is_error=failed_nums, + fg="red" if failed_nums else "green", + ) diff --git a/platformio/check/defect.py b/platformio/check/defect.py new file mode 100644 index 0000000..5689a18 --- /dev/null +++ b/platformio/check/defect.py @@ -0,0 +1,95 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click + +from platformio.exception import PlatformioException +from platformio.project.helpers import get_project_dir + +# pylint: disable=too-many-instance-attributes, redefined-builtin +# pylint: disable=too-many-arguments + + +class DefectItem: + SEVERITY_HIGH = 1 + SEVERITY_MEDIUM = 2 + SEVERITY_LOW = 4 + SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"} + + def __init__( + self, + severity, + category, + message, + file=None, + line=0, + column=0, + id=None, + callstack=None, + cwe=None, + ): + assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW) + self.severity = severity + self.category = category + self.message = message + self.line = int(line) + self.column = int(column) + self.callstack = callstack + self.cwe = cwe + self.id = id + self.file = file or "unknown" + if file.lower().startswith(get_project_dir().lower()): + self.file = os.path.relpath(file, get_project_dir()) + + def __repr__(self): + defect_color = None + if self.severity == self.SEVERITY_HIGH: + defect_color = "red" + elif self.severity == self.SEVERITY_MEDIUM: + defect_color = "yellow" + + format_str = "{file}:{line}: [{severity}:{category}] {message} {id}" + return format_str.format( + severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color), + category=click.style(self.category.lower(), fg=defect_color), + file=click.style(self.file, bold=True), + message=self.message, + line=self.line, + id="%s" % "[%s]" % self.id if self.id else "", + ) + + def __or__(self, defect): + return self.severity | defect.severity + + @staticmethod + def severity_to_int(label): + for key, value in DefectItem.SEVERITY_LABELS.items(): + if label == value: + return key + raise PlatformioException("Unknown severity label -> %s" % label) + + def as_dict(self): + return { + "severity": self.SEVERITY_LABELS[self.severity], + "category": self.category, + "message": self.message, + "file": os.path.abspath(self.file), + "line": self.line, + "column": self.column, + "callstack": self.callstack, + "id": self.id, + "cwe": self.cwe, + } diff --git a/platformio/check/tools/__init__.py b/platformio/check/tools/__init__.py new file mode 100644 index 0000000..58a9263 --- /dev/null +++ b/platformio/check/tools/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import exception +from platformio.check.tools.clangtidy import ClangtidyCheckTool +from platformio.check.tools.cppcheck import CppcheckCheckTool +from platformio.check.tools.pvsstudio import PvsStudioCheckTool + + +class CheckToolFactory: + @staticmethod + def new(tool, project_dir, config, envname, options): + cls = None + if tool == "cppcheck": + cls = CppcheckCheckTool + elif tool == "clangtidy": + cls = ClangtidyCheckTool + elif tool == "pvs-studio": + cls = PvsStudioCheckTool + else: + raise exception.PlatformioException("Unknown check tool `%s`" % tool) + return cls(project_dir, config, envname, options) diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py new file mode 100644 index 0000000..ccf331f --- /dev/null +++ b/platformio/check/tools/base.py @@ -0,0 +1,267 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile + +import click + +from platformio import fs, proc +from platformio.check.defect import DefectItem +from platformio.package.manager.core import get_core_package_dir +from platformio.package.meta import PackageSpec +from platformio.project.helpers import load_build_metadata + + +class CheckToolBase: # pylint: disable=too-many-instance-attributes + def __init__(self, project_dir, config, envname, options): + self.config = config + self.envname = envname + self.options = options + self.project_dir = project_dir + self.cc_flags = [] + self.cxx_flags = [] + self.cpp_includes = [] + self.cpp_defines = [] + self.toolchain_defines = [] + self._tmp_files = [] + self.cc_path = None + self.cxx_path = None + self._defects = [] + self._on_defect_callback = None + self._bad_input = False + self._load_cpp_data() + + # detect all defects by default + if not self.options.get("severity"): + self.options["severity"] = [ + DefectItem.SEVERITY_LOW, + DefectItem.SEVERITY_MEDIUM, + DefectItem.SEVERITY_HIGH, + ] + # cast to severity by ids + self.options["severity"] = [ + s if isinstance(s, int) else DefectItem.severity_to_int(s) + for s in self.options["severity"] + ] + + def _load_cpp_data(self): + data = load_build_metadata(self.project_dir, self.envname) + if not data: + return + self.cc_flags = data.get("cc_flags", []) + self.cxx_flags = data.get("cxx_flags", []) + self.cpp_includes = self._dump_includes(data.get("includes", {})) + self.cpp_defines = data.get("defines", []) + self.cc_path = data.get("cc_path") + self.cxx_path = data.get("cxx_path") + self.toolchain_defines = self._get_toolchain_defines() + + def get_tool_dir(self, pkg_name): + for spec in self.options["platform_packages"] or []: + spec = PackageSpec(spec) + if spec.name == pkg_name: + return get_core_package_dir(pkg_name, spec=spec) + return get_core_package_dir(pkg_name) + + def get_flags(self, tool): + result = [] + flags = self.options.get("flags") or [] + for flag in flags: + if ":" not in flag or flag.startswith("-"): + result.extend([f for f in flag.split(" ") if f]) + elif flag.startswith("%s:" % tool): + result.extend([f for f in flag.split(":", 1)[1].split(" ") if f]) + + return result + + def _get_toolchain_defines(self): + def _extract_defines(language, includes_file): + build_flags = self.cxx_flags if language == "c++" else self.cc_flags + defines = [] + cmd = "echo | %s -x %s %s %s -dM -E -" % ( + self.cc_path, + language, + " ".join( + [f for f in build_flags if f.startswith(("-m", "-f", "-std"))] + ), + includes_file, + ) + result = proc.exec_command(cmd, shell=True) + + if result["returncode"] != 0: + click.echo("Warning: Failed to extract toolchain defines!") + if self.options.get("verbose"): + click.echo(result["out"]) + click.echo(result["err"]) + + for line in result["out"].split("\n"): + tokens = line.strip().split(" ", 2) + if not tokens or tokens[0] != "#define": + continue + if len(tokens) > 2: + defines.append("%s=%s" % (tokens[1], tokens[2])) + else: + defines.append(tokens[1]) + + return defines + + incflags_file = self._long_includes_hook(self.cpp_includes) + return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")} + + def _create_tmp_file(self, data): + with tempfile.NamedTemporaryFile("w", delete=False) as fp: + fp.write(data) + self._tmp_files.append(fp.name) + return fp.name + + def _long_includes_hook(self, includes): + data = [] + for inc in includes: + data.append('-I"%s"' % fs.to_unix_path(inc)) + + return '@"%s"' % self._create_tmp_file(" ".join(data)) + + @staticmethod + def _dump_includes(includes_map): + result = [] + for includes in includes_map.values(): + for include in includes: + if include not in result: + result.append(include) + return result + + @staticmethod + def is_flag_set(flag, flags): + return any(flag in f for f in flags) + + def get_defects(self): + return self._defects + + def configure_command(self): + raise NotImplementedError + + def on_tool_output(self, line): + line = self.tool_output_filter(line) + if not line: + return + + defect = self.parse_defect(line) + + if not isinstance(defect, DefectItem): + if self.options.get("verbose"): + click.echo(line) + return + + if defect.severity not in self.options["severity"]: + return + + self._defects.append(defect) + if self._on_defect_callback: + self._on_defect_callback(defect) + + @staticmethod + def tool_output_filter(line): + return line + + @staticmethod + def parse_defect(raw_line): + return raw_line + + def clean_up(self): + for f in self._tmp_files: + if os.path.isfile(f): + os.remove(f) + + @staticmethod + def is_check_successful(cmd_result): + return cmd_result["returncode"] == 0 + + def execute_check_cmd(self, cmd): + result = proc.exec_command( + cmd, + stdout=proc.LineBufferedAsyncPipe(self.on_tool_output), + stderr=proc.LineBufferedAsyncPipe(self.on_tool_output), + ) + + if not self.is_check_successful(result): + click.echo( + "\nError: Failed to execute check command! Exited with code %d." + % result["returncode"] + ) + if self.options.get("verbose"): + click.echo(result["out"]) + click.echo(result["err"]) + self._bad_input = True + + return result + + @staticmethod + def get_project_target_files(project_dir, src_filters): + c_extension = (".c",) + cpp_extensions = (".cc", ".cpp", ".cxx", ".ino") + header_extensions = (".h", ".hh", ".hpp", ".hxx") + + result = {"c": [], "c++": [], "headers": []} + + def _add_file(path): + if path.endswith(header_extensions): + result["headers"].append(os.path.abspath(path)) + elif path.endswith(c_extension): + result["c"].append(os.path.abspath(path)) + elif path.endswith(cpp_extensions): + result["c++"].append(os.path.abspath(path)) + + src_filters = normalize_src_filters(src_filters) + for f in fs.match_src_files(project_dir, src_filters): + _add_file(f) + + return result + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + cmd = self.configure_command() + if cmd: + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + + self.execute_check_cmd(cmd) + + else: + if self.options.get("verbose"): + click.echo("Error: Couldn't configure command") + self._bad_input = True + + self.clean_up() + + return self._bad_input + + +# +# Helpers +# + + +def normalize_src_filters(src_filters): + def _normalize(src_filters): + return ( + src_filters + if src_filters.startswith(("+<", "-<")) + else "+<%s>" % src_filters + ) + + if isinstance(src_filters, (list, tuple)): + return " ".join([_normalize(f) for f in src_filters]) + + return _normalize(src_filters) diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py new file mode 100644 index 0000000..496acc7 --- /dev/null +++ b/platformio/check/tools/clangtidy.py @@ -0,0 +1,90 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from os.path import join + +from platformio.check.defect import DefectItem +from platformio.check.tools.base import CheckToolBase + + +class ClangtidyCheckTool(CheckToolBase): + def tool_output_filter(self, line): # pylint: disable=arguments-differ + if not self.options.get("verbose") and "[clang-diagnostic-error]" in line: + return "" + + if "[CommonOptionsParser]" in line: + self._bad_input = True + return line + + if any(d in line for d in ("note: ", "error: ", "warning: ")): + return line + + return "" + + def parse_defect(self, raw_line): # pylint: disable=arguments-differ + match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line) + if not match: + return raw_line + + file_, line, column, category, message, defect_id = match.groups() + + severity = DefectItem.SEVERITY_LOW + if category == "error": + severity = DefectItem.SEVERITY_HIGH + elif category == "warning": + severity = DefectItem.SEVERITY_MEDIUM + + return DefectItem(severity, category, message, file_, line, column, defect_id) + + @staticmethod + def is_check_successful(cmd_result): + # Note: Clang-Tidy returns 1 for not critical compilation errors, + # so 0 and 1 are only acceptable values + return cmd_result["returncode"] < 2 + + def configure_command(self): + tool_path = join(self.get_tool_dir("tool-clangtidy"), "clang-tidy") + + cmd = [tool_path, "--quiet"] + flags = self.get_flags("clangtidy") + if not ( + self.is_flag_set("--checks", flags) or self.is_flag_set("--config", flags) + ): + cmd.append("--checks=*") + + project_files = self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ) + + src_files = [] + for items in project_files.values(): + src_files.extend(items) + + cmd.extend(flags + src_files + ["--"]) + cmd.extend( + ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines["c++"]] + ) + + includes = [] + for inc in self.cpp_includes: + if self.options.get("skip_packages") and inc.lower().startswith( + self.config.get("platformio", "packages_dir").lower() + ): + continue + includes.append(inc) + + cmd.extend(["-I%s" % inc for inc in includes]) + + return cmd diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py new file mode 100644 index 0000000..c3c5953 --- /dev/null +++ b/platformio/check/tools/cppcheck.py @@ -0,0 +1,271 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click + +from platformio import proc +from platformio.check.defect import DefectItem +from platformio.check.tools.base import CheckToolBase + + +class CppcheckCheckTool(CheckToolBase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._field_delimiter = "<&PIO&>" + self._buffer = "" + self.defect_fields = [ + "severity", + "message", + "file", + "line", + "column", + "callstack", + "cwe", + "id", + ] + + def tool_output_filter(self, line): # pylint: disable=arguments-differ + if ( + not self.options.get("verbose") + and "--suppress=unmatchedSuppression:" in line + ): + return "" + + if any( + msg in line + for msg in ( + "No C or C++ source files found", + "unrecognized command line option", + "there was an internal error", + ) + ): + self._bad_input = True + + return line + + def parse_defect(self, raw_line): # pylint: disable=arguments-differ + if self._field_delimiter not in raw_line: + return None + + self._buffer += raw_line + if any(f not in self._buffer for f in self.defect_fields): + return None + + args = {} + for field in self._buffer.split(self._field_delimiter): + field = field.strip().replace('"', "") + name, value = field.split("=", 1) + args[name] = value + + args["category"] = args["severity"] + if args["severity"] == "error": + args["severity"] = DefectItem.SEVERITY_HIGH + elif args["severity"] == "warning": + args["severity"] = DefectItem.SEVERITY_MEDIUM + else: + args["severity"] = DefectItem.SEVERITY_LOW + + # Skip defects found in third-party software, but keep in mind that such defects + # might break checking process so defects from project files are not reported + breaking_defect_ids = ("preprocessorErrorDirective", "syntaxError") + if ( + args.get("file", "") + .lower() + .startswith(self.config.get("platformio", "packages_dir").lower()) + ): + if args["id"] in breaking_defect_ids: + if self.options.get("verbose"): + click.echo( + "Error: Found a breaking defect '%s' in %s:%s\n" + "Please note: check results might not be valid!\n" + "Try adding --skip-packages" + % (args.get("message"), args.get("file"), args.get("line")) + ) + click.echo() + self._bad_input = True + self._buffer = "" + return None + + self._buffer = "" + return DefectItem(**args) + + def configure_command(self, language, src_file): # pylint: disable=arguments-differ + tool_path = os.path.join(self.get_tool_dir("tool-cppcheck"), "cppcheck") + + cmd = [ + tool_path, + "--addon-python=%s" % proc.get_pythonexe_path(), + "--error-exitcode=3", + "--verbose" if self.options.get("verbose") else "--quiet", + ] + + cmd.append( + '--template="%s"' + % self._field_delimiter.join( + ["{0}={{{0}}}".format(f) for f in self.defect_fields] + ) + ) + + flags = self.get_flags("cppcheck") + if not flags: + # by default user can suppress reporting individual defects + # directly in code // cppcheck-suppress warningID + cmd.append("--inline-suppr") + if not self.is_flag_set("--platform", flags): + cmd.append("--platform=unspecified") + if not self.is_flag_set("--enable", flags): + enabled_checks = [ + "warning", + "style", + "performance", + "portability", + "unusedFunction", + ] + cmd.append("--enable=%s" % ",".join(enabled_checks)) + + if not self.is_flag_set("--language", flags): + cmd.append("--language=" + language) + + build_flags = self.cxx_flags if language == "c++" else self.cc_flags + + if not self.is_flag_set("--std", flags): + # Try to guess the standard version from the build flags + for flag in build_flags: + if "-std" in flag: + cmd.append("-" + self.convert_language_standard(flag)) + + cmd.extend( + ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]] + ) + + cmd.extend(flags) + + cmd.extend( + "--include=" + inc + for inc in self.get_forced_includes(build_flags, self.cpp_includes) + ) + cmd.append("--includes-file=%s" % self._generate_inc_file()) + cmd.append('"%s"' % src_file) + + return cmd + + @staticmethod + def get_forced_includes(build_flags, includes): + def _extract_filepath(flag, include_options, build_flags): + path = "" + for option in include_options: + if not flag.startswith(option): + continue + if flag.split(option)[1].strip(): + path = flag.split(option)[1].strip() + elif build_flags.index(flag) + 1 < len(build_flags): + path = build_flags[build_flags.index(flag) + 1] + return path + + def _search_include_dir(filepath, include_paths): + for inc_path in include_paths: + path = os.path.join(inc_path, filepath) + if os.path.isfile(path): + return path + return "" + + result = [] + include_options = ("-include", "-imacros") + for f in build_flags: + if f.startswith(include_options): + filepath = _extract_filepath(f, include_options, build_flags) + if not os.path.isabs(filepath): + filepath = _search_include_dir(filepath, includes) + if os.path.isfile(filepath): + result.append(filepath) + + return result + + def _generate_src_file(self, src_files): + return self._create_tmp_file("\n".join(src_files)) + + def _generate_inc_file(self): + result = [] + for inc in self.cpp_includes: + if self.options.get("skip_packages") and inc.lower().startswith( + self.config.get("platformio", "packages_dir").lower() + ): + continue + result.append(inc) + return self._create_tmp_file("\n".join(result)) + + def clean_up(self): + super().clean_up() + + # delete temporary dump files generated by addons + if not self.is_flag_set("--addon", self.get_flags("cppcheck")): + return + + for files in self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ).values(): + for f in files: + dump_file = f + ".dump" + if os.path.isfile(dump_file): + os.remove(dump_file) + + @staticmethod + def is_check_successful(cmd_result): + # Cppcheck is configured to return '3' if a defect is found + return cmd_result["returncode"] in (0, 3) + + @staticmethod + def convert_language_standard(flag): + cpp_standards_map = { + "0x": "11", + "1y": "14", + "1z": "17", + "2a": "20", + } + + standard = flag[-2:] + # Note: GNU extensions are not supported and converted to regular standards + return flag.replace("gnu", "c").replace( + standard, cpp_standards_map.get(standard, standard) + ) + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + + project_files = self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ) + src_files_scope = ("c", "c++") + if not any(project_files[t] for t in src_files_scope): + click.echo("Error: Nothing to check.") + return True + + for scope, files in project_files.items(): + if scope not in src_files_scope: + continue + for src_file in files: + cmd = self.configure_command(scope, src_file) + if not cmd: + self._bad_input = True + continue + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + + self.execute_check_cmd(cmd) + + self.clean_up() + + return self._bad_input diff --git a/platformio/check/tools/pvsstudio.py b/platformio/check/tools/pvsstudio.py new file mode 100644 index 0000000..20979a0 --- /dev/null +++ b/platformio/check/tools/pvsstudio.py @@ -0,0 +1,251 @@ +# Copyright (c) 2020-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile +from xml.etree.ElementTree import fromstring + +import click + +from platformio import proc +from platformio.check.defect import DefectItem +from platformio.check.tools.base import CheckToolBase +from platformio.compat import IS_WINDOWS + + +class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-attributes + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._tmp_dir = tempfile.mkdtemp(prefix="piocheck") + self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i" + self._tmp_output_file = self._generate_tmp_file_path() + ".pvs" + self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg" + self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd" + self.tool_path = os.path.join( + self.get_tool_dir("tool-pvs-studio"), + "x64" if IS_WINDOWS else "bin", + "pvs-studio", + ) + + with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp: + fp.write( + "exclude-path = " + + self.config.get("platformio", "packages_dir").replace("\\", "/") + ) + + with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp: + fp.write( + " ".join( + ['-I"%s"' % inc.replace("\\", "/") for inc in self.cpp_includes] + ) + ) + + def tool_output_filter(self, line): # pylint: disable=arguments-differ + if any( + err_msg in line.lower() + for err_msg in ( + "license was not entered", + "license information is incorrect", + ) + ): + self._bad_input = True + return line + + def _process_defects(self, defects): + for defect in defects: + if not isinstance(defect, DefectItem): + return + if defect.severity not in self.options["severity"]: + return + self._defects.append(defect) + if self._on_defect_callback: + self._on_defect_callback(defect) + + def _demangle_report(self, output_file): + converter_tool = os.path.join( + self.get_tool_dir("tool-pvs-studio"), + "HtmlGenerator" if IS_WINDOWS else os.path.join("bin", "plog-converter"), + ) + + cmd = ( + converter_tool, + "-t", + "xml", + output_file, + "-m", + "cwe", + "-m", + "misra", + "-a", + # Enable all possible analyzers and defect levels + "GA:1,2,3;64:1,2,3;OP:1,2,3;CS:1,2,3;MISRA:1,2,3", + "--cerr", + ) + + result = proc.exec_command(cmd) + if result["returncode"] != 0: + click.echo(result["err"]) + self._bad_input = True + + return result["err"] + + def parse_defects(self, output_file): + defects = [] + + report = self._demangle_report(output_file) + if not report: + self._bad_input = True + return [] + + try: + defects_data = fromstring(report) + except: # pylint: disable=bare-except + click.echo("Error: Couldn't decode generated report!") + self._bad_input = True + return [] + + for table in defects_data.iter("PVS-Studio_Analysis_Log"): + message = table.find("Message").text + category = table.find("ErrorType").text + line = table.find("Line").text + file_ = table.find("File").text + defect_id = table.find("ErrorCode").text + cwe = table.find("CWECode") + cwe_id = None + if cwe is not None: + cwe_id = cwe.text.lower().replace("cwe-", "") + misra = table.find("MISRA") + if misra is not None: + message += " [%s]" % misra.text + + severity = DefectItem.SEVERITY_LOW + if category == "error": + severity = DefectItem.SEVERITY_HIGH + elif category == "warning": + severity = DefectItem.SEVERITY_MEDIUM + + defects.append( + DefectItem( + severity, category, message, file_, line, id=defect_id, cwe=cwe_id + ) + ) + + return defects + + def configure_command(self, src_file): # pylint: disable=arguments-differ + if os.path.isfile(self._tmp_output_file): + os.remove(self._tmp_output_file) + + if not os.path.isfile(self._tmp_preprocessed_file): + click.echo("Error: Missing preprocessed file for '%s'" % src_file) + return "" + + cmd = [ + self.tool_path, + "--skip-cl-exe", + "yes", + "--language", + "C" if src_file.endswith(".c") else "C++", + "--preprocessor", + "gcc", + "--cfg", + self._tmp_cfg_file, + "--source-file", + src_file, + "--i-file", + self._tmp_preprocessed_file, + "--output-file", + self._tmp_output_file, + ] + + flags = self.get_flags("pvs-studio") + if not self.is_flag_set("--platform", flags): + cmd.append("--platform=arm") + cmd.extend(flags) + + return cmd + + def _generate_tmp_file_path(self): + # pylint: disable=protected-access + return os.path.join(self._tmp_dir, next(tempfile._get_candidate_names())) + + def _prepare_preprocessed_file(self, src_file): + if os.path.isfile(self._tmp_preprocessed_file): + os.remove(self._tmp_preprocessed_file) + + flags = self.cxx_flags + compiler = self.cxx_path + if src_file.endswith(".c"): + flags = self.cc_flags + compiler = self.cc_path + + cmd = [ + compiler, + '"%s"' % src_file, + "-E", + "-o", + '"%s"' % self._tmp_preprocessed_file, + ] + cmd.extend([f for f in flags if f]) + cmd.extend(['"-D%s"' % d.replace('"', '\\"') for d in self.cpp_defines]) + cmd.append('@"%s"' % self._tmp_cmd_file) + + # Explicitly specify C++ as the language used in .ino files + if src_file.endswith(".ino"): + cmd.insert(1, "-xc++") + + result = proc.exec_command(" ".join(cmd), shell=True) + if result["returncode"] != 0 or result["err"]: + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + click.echo(result["err"]) + self._bad_input = True + + def clean_up(self): + super().clean_up() + if os.path.isdir(self._tmp_dir): + shutil.rmtree(self._tmp_dir) + + @staticmethod + def is_check_successful(cmd_result): + return ( + "license" not in cmd_result["err"].lower() and cmd_result["returncode"] == 0 + ) + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + for scope, files in self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ).items(): + if scope not in ("c", "c++"): + continue + for src_file in files: + self._prepare_preprocessed_file(src_file) + cmd = self.configure_command(src_file) + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + if not cmd: + self._bad_input = True + continue + + result = self.execute_check_cmd(cmd) + if result["returncode"] != 0: + continue + + self._process_defects(self.parse_defects(self._tmp_output_file)) + + self.clean_up() + + return self._bad_input diff --git a/platformio/cli.py b/platformio/cli.py new file mode 100644 index 0000000..a2a707a --- /dev/null +++ b/platformio/cli.py @@ -0,0 +1,110 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from pathlib import Path + +import click + + +class PlatformioCLI(click.MultiCommand): + leftover_args = [] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._pio_root_path = Path(__file__).parent + self._pio_cmd_aliases = dict(package="pkg") + + def _find_pio_commands(self): + def _to_module_path(p): + return ( + "platformio." + ".".join(p.relative_to(self._pio_root_path).parts)[:-3] + ) + + result = {} + for p in self._pio_root_path.rglob("cli.py"): + # skip this module + if p.parent == self._pio_root_path: + continue + cmd_name = p.parent.name + result[self._pio_cmd_aliases.get(cmd_name, cmd_name)] = _to_module_path(p) + + # find legacy commands + for p in (self._pio_root_path / "commands").iterdir(): + if p.name.startswith("_"): + continue + if (p / "command.py").is_file(): + result[p.name] = _to_module_path(p / "command.py") + elif p.name.endswith(".py"): + result[p.name[:-3]] = _to_module_path(p) + + return result + + @staticmethod + def in_silence(): + args = PlatformioCLI.leftover_args + return args and any( + [ + args[0] == "debug" and "--interpreter" in " ".join(args), + args[0] == "upgrade", + "--json-output" in args, + "--version" in args, + ] + ) + + @classmethod + def reveal_cmd_path_args(cls, ctx): + result = [] + group = ctx.command + args = cls.leftover_args[::] + while args: + cmd_name = args.pop(0) + next_group = group.get_command(ctx, cmd_name) + if next_group: + group = next_group + result.append(cmd_name) + if not hasattr(group, "get_command"): + break + return result + + def invoke(self, ctx): + PlatformioCLI.leftover_args = ctx.args + if hasattr(ctx, "protected_args"): + PlatformioCLI.leftover_args = ctx.protected_args + ctx.args + return super().invoke(ctx) + + def list_commands(self, ctx): + return sorted(list(self._find_pio_commands())) + + def get_command(self, ctx, cmd_name): + commands = self._find_pio_commands() + if cmd_name not in commands: + return self._handle_obsolate_command(ctx, cmd_name) + module = importlib.import_module(commands[cmd_name]) + return getattr(module, "cli") + + @staticmethod + def _handle_obsolate_command(ctx, cmd_name): + # pylint: disable=import-outside-toplevel + if cmd_name == "init": + from platformio.project.commands.init import project_init_cmd + + return project_init_cmd + + if cmd_name == "package": + from platformio.package.cli import cli + + return cli + + raise click.UsageError('No such command "%s"' % cmd_name, ctx) diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py new file mode 100644 index 0000000..ac8367d --- /dev/null +++ b/platformio/commands/boards.py @@ -0,0 +1,86 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import shutil + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.package.manager.platform import PlatformPackageManager + + +@click.command("boards", short_help="Board Explorer") +@click.argument("query", required=False) +@click.option("--installed", is_flag=True) +@click.option("--json-output", is_flag=True) +def cli(query, installed, json_output): # pylint: disable=R0912 + if json_output: + return _print_boards_json(query, installed) + + grpboards = {} + for board in _get_boards(installed): + if query and not any( + query.lower() in str(board.get(k, "")).lower() + for k in ("id", "name", "mcu", "vendor", "platform", "frameworks") + ): + continue + if board["platform"] not in grpboards: + grpboards[board["platform"]] = [] + grpboards[board["platform"]].append(board) + + terminal_width = shutil.get_terminal_size().columns + for platform, boards in sorted(grpboards.items()): + click.echo("") + click.echo("Platform: ", nl=False) + click.secho(platform, bold=True) + click.echo("=" * terminal_width) + print_boards(boards) + return True + + +def print_boards(boards): + click.echo( + tabulate( + [ + ( + click.style(b["id"], fg="cyan"), + b["mcu"], + "%dMHz" % (b["fcpu"] / 1000000), + fs.humanize_file_size(b["rom"]), + fs.humanize_file_size(b["ram"]), + b["name"], + ) + for b in boards + ], + headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"], + ) + ) + + +def _get_boards(installed=False): + pm = PlatformPackageManager() + return pm.get_installed_boards() if installed else pm.get_all_boards() + + +def _print_boards_json(query, installed=False): + result = [] + for board in _get_boards(installed): + if query: + search_data = "%s %s" % (board["id"], json.dumps(board).lower()) + if query.lower() not in search_data.lower(): + continue + result.append(board) + click.echo(json.dumps(result)) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py new file mode 100644 index 0000000..5be3128 --- /dev/null +++ b/platformio/commands/ci.py @@ -0,0 +1,174 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os +import shutil +import tempfile + +import click + +from platformio import fs +from platformio.exception import CIBuildEnvsEmpty +from platformio.project.commands.init import project_init_cmd, validate_boards +from platformio.project.config import ProjectConfig +from platformio.run.cli import cli as cmd_run + + +def validate_path(ctx, param, value): # pylint: disable=unused-argument + invalid_path = None + value = list(value) + for i, p in enumerate(value): + if p.startswith("~"): + value[i] = fs.expanduser(p) + value[i] = os.path.abspath(value[i]) + if not glob.glob(value[i], recursive=True): + invalid_path = p + break + try: + assert invalid_path is None + return value + except AssertionError as exc: + raise click.BadParameter("Found invalid path: %s" % invalid_path) from exc + + +@click.command("ci", short_help="Continuous Integration") +@click.argument("src", nargs=-1, callback=validate_path) +@click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY") +@click.option("--exclude", multiple=True) +@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) +@click.option( + "--build-dir", + default=tempfile.mkdtemp, + type=click.Path(file_okay=False, dir_okay=True, writable=True), +) +@click.option("--keep-build-dir", is_flag=True) +@click.option( + "-c", + "--project-conf", + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), +) +@click.option("-O", "--project-option", multiple=True) +@click.option("-e", "--environment", "environments", multiple=True) +@click.option("-v", "--verbose", is_flag=True) +@click.pass_context +def cli( # pylint: disable=too-many-arguments, too-many-branches + ctx, + src, + lib, + exclude, + board, + build_dir, + keep_build_dir, + project_conf, + project_option, + environments, + verbose, +): + if not src and os.getenv("PLATFORMIO_CI_SRC"): + src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":")) + if not src: + raise click.BadParameter("Missing argument 'src'") + + try: + if not keep_build_dir and os.path.isdir(build_dir): + fs.rmtree(build_dir) + if not os.path.isdir(build_dir): + os.makedirs(build_dir) + + for dir_name, patterns in dict(lib=lib, src=src).items(): + if not patterns: + continue + contents = [] + for p in patterns: + contents += glob.glob(p, recursive=True) + _copy_contents(os.path.join(build_dir, dir_name), contents) + + if project_conf and os.path.isfile(project_conf): + _copy_project_conf(build_dir, project_conf) + elif not board: + raise CIBuildEnvsEmpty() + + if exclude: + _exclude_contents(build_dir, exclude) + + # initialise project + ctx.invoke( + project_init_cmd, + project_dir=build_dir, + boards=board, + project_options=project_option, + ) + + # process project + ctx.invoke( + cmd_run, project_dir=build_dir, environment=environments, verbose=verbose + ) + finally: + if not keep_build_dir: + fs.rmtree(build_dir) + + +def _copy_contents(dst_dir, contents): # pylint: disable=too-many-branches + items = {"dirs": set(), "files": set()} + + for path in contents: + if os.path.isdir(path): + items["dirs"].add(path) + elif os.path.isfile(path): + items["files"].add(path) + + dst_dir_name = os.path.basename(dst_dir) + + if dst_dir_name == "src" and len(items["dirs"]) == 1: + if not os.path.isdir(dst_dir): + shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) + else: + if not os.path.isdir(dst_dir): + os.makedirs(dst_dir) + for d in items["dirs"]: + src_dst_dir = os.path.join(dst_dir, os.path.basename(d)) + if not os.path.isdir(src_dst_dir): + shutil.copytree(d, src_dst_dir, symlinks=True) + + if not items["files"]: + return + + if dst_dir_name == "lib": + dst_dir = os.path.join(dst_dir, tempfile.mkdtemp(dir=dst_dir)) + + for f in items["files"]: + dst_file = os.path.join(dst_dir, os.path.basename(f)) + if f == dst_file: + continue + shutil.copyfile(f, dst_file) + + +def _exclude_contents(dst_dir, patterns): + contents = [] + for p in patterns: + contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True) + for path in contents: + path = os.path.abspath(path) + if os.path.isdir(path): + fs.rmtree(path) + elif os.path.isfile(path): + os.remove(path) + + +def _copy_project_conf(build_dir, project_conf): + config = ProjectConfig(project_conf, parse_extra=False) + if config.has_section("platformio"): + config.remove_section("platformio") + config.save(os.path.join(build_dir, "platformio.ini")) diff --git a/platformio/commands/device/__init__.py b/platformio/commands/device/__init__.py new file mode 100644 index 0000000..d78b003 --- /dev/null +++ b/platformio/commands/device/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-import +from platformio.device.monitor.filters.base import ( + DeviceMonitorFilterBase as DeviceMonitorFilter, +) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py new file mode 100644 index 0000000..1c150c0 --- /dev/null +++ b/platformio/commands/lib.py @@ -0,0 +1,459 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-branches, too-many-locals + +import json +import logging +import os + +import click + +from platformio import exception, fs +from platformio.cli import PlatformioCLI +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.list import package_list_cmd +from platformio.package.commands.search import package_search_cmd +from platformio.package.commands.show import package_show_cmd +from platformio.package.commands.uninstall import package_uninstall_cmd +from platformio.package.commands.update import package_update_cmd +from platformio.package.exception import NotGlobalLibDir +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.meta import PackageItem, PackageSpec +from platformio.proc import is_ci +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_dir, is_platformio_project + +CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs" +CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments" +CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs" +CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" + + +def get_project_global_lib_dir(): + return ProjectConfig.get_instance().get("platformio", "globallib_dir") + + +def invoke_command(ctx, cmd, **kwargs): + input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) + project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] + for input_dir in input_dirs: + cmd_kwargs = kwargs.copy() + if is_platformio_project(input_dir): + cmd_kwargs["project_dir"] = input_dir + cmd_kwargs["environments"] = project_environments + else: + cmd_kwargs["global"] = True + cmd_kwargs["storage_dir"] = input_dir + ctx.invoke(cmd, **cmd_kwargs) + + +@click.group(short_help="Library manager", hidden=True) +@click.option( + "-d", + "--storage-dir", + multiple=True, + default=None, + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), + help="Manage custom library storage", +) +@click.option( + "-g", "--global", is_flag=True, help="Manage global PlatformIO library storage" +) +@click.option( + "-e", + "--environment", + multiple=True, + help=( + "Manage libraries for the specific project build environments " + "declared in `platformio.ini`" + ), +) +@click.pass_context +def cli(ctx, **options): + in_silence = PlatformioCLI.in_silence() + storage_cmds = ("install", "uninstall", "update", "list") + # skip commands that don't need storage folder + if ctx.invoked_subcommand not in storage_cmds or ( + len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help") + ): + return + storage_dirs = list(options["storage_dir"]) + if options["global"]: + storage_dirs.append(get_project_global_lib_dir()) + if not storage_dirs: + if is_platformio_project(): + storage_dirs = [get_project_dir()] + elif is_ci(): + storage_dirs = [get_project_global_lib_dir()] + click.secho( + "Warning! Global library storage is used automatically. " + "Please use `platformio lib --global %s` command to remove " + "this warning." % ctx.invoked_subcommand, + fg="yellow", + ) + + if not storage_dirs: + raise NotGlobalLibDir( + get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand + ) + + ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options["environment"] + ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [] + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {} + for storage_dir in storage_dirs: + if not is_platformio_project(storage_dir): + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) + continue + with fs.cd(storage_dir): + config = ProjectConfig.get_instance( + os.path.join(storage_dir, "platformio.ini") + ) + config.validate(options["environment"], silent=in_silence) + libdeps_dir = config.get("platformio", "libdeps_dir") + for env in config.envs(): + if options["environment"] and env not in options["environment"]: + continue + storage_dir = os.path.join(libdeps_dir, env) + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( + "env:" + env, "lib_deps", [] + ) + + +@cli.command("install", short_help="Install library") +@click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") +@click.option( + "--save/--no-save", + is_flag=True, + default=True, + help="Save installed libraries into the `platformio.ini` dependency list" + " (enabled by default)", +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +@click.option( + "--interactive", + is_flag=True, + help="Deprecated! Please use a strict dependency specification (owner/libname)", +) +@click.option( + "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists" +) +@click.pass_context +def lib_install( # pylint: disable=too-many-arguments,unused-argument + ctx, libraries, save, silent, interactive, force +): + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg install` instead.\n", + fg="yellow", + ) + return invoke_command( + ctx, + package_install_cmd, + libraries=libraries, + no_save=not save, + force=force, + silent=silent, + ) + + +@cli.command("uninstall", short_help="Remove libraries") +@click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") +@click.option( + "--save/--no-save", + is_flag=True, + default=True, + help="Remove libraries from the `platformio.ini` dependency list and save changes" + " (enabled by default)", +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +@click.pass_context +def lib_uninstall(ctx, libraries, save, silent): + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg uninstall` instead.\n", + fg="yellow", + ) + invoke_command( + ctx, + package_uninstall_cmd, + libraries=libraries, + no_save=not save, + silent=silent, + ) + + +@cli.command("update", short_help="Update installed libraries") +@click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") +@click.option( + "-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead", +) +@click.option( + "--dry-run", is_flag=True, help="Do not update, only check for the new versions" +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +@click.option("--json-output", is_flag=True) +@click.pass_context +def lib_update( # pylint: disable=too-many-arguments + ctx, libraries, only_check, dry_run, silent, json_output +): + only_check = dry_run or only_check + if only_check and not json_output: + raise exception.UserSideException( + "This command is deprecated, please use `pio pkg outdated` instead" + ) + + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg update` instead.\n", + fg="yellow", + ) + return invoke_command( + ctx, + package_update_cmd, + libraries=libraries, + silent=silent, + ) + + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + json_result = {} + for storage_dir in storage_dirs: + lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, []) + lm = LibraryPackageManager(storage_dir) + lm.set_log_level(logging.WARN if silent else logging.DEBUG) + _libraries = libraries or lib_deps or lm.get_installed() + + result = [] + for library in _libraries: + spec = None + pkg = None + if isinstance(library, PackageItem): + pkg = library + else: + spec = PackageSpec(library) + pkg = lm.get_package(spec) + if not pkg: + continue + outdated = lm.outdated(pkg, spec) + if not outdated.is_outdated(allow_incompatible=True): + continue + manifest = lm.legacy_load_manifest(pkg) + manifest["versionWanted"] = ( + str(outdated.wanted) if outdated.wanted else None + ) + manifest["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) + result.append(manifest) + + json_result[storage_dir] = result + + return click.echo( + json.dumps( + json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result + ) + ) + + +@cli.command("list", short_help="List installed libraries") +@click.option("--json-output", is_flag=True) +@click.pass_context +def lib_list(ctx, json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg list` instead.\n", + fg="yellow", + ) + return invoke_command(ctx, package_list_cmd, only_libraries=True) + + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + json_result = {} + for storage_dir in storage_dirs: + lm = LibraryPackageManager(storage_dir) + json_result[storage_dir] = lm.legacy_get_installed() + return click.echo( + json.dumps( + json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result + ) + ) + + +@cli.command("search", short_help="Search for a library") +@click.argument("query", required=False, nargs=-1) +@click.option("--json-output", is_flag=True) +@click.option("--page", type=click.INT, default=1) +@click.option("--id", multiple=True) +@click.option("-o", "--owner", multiple=True) +@click.option("-n", "--name", multiple=True) +@click.option("-a", "--author", multiple=True) +@click.option("-k", "--keyword", multiple=True) +@click.option("-f", "--framework", multiple=True) +@click.option("-p", "--platform", multiple=True) +@click.option("-i", "--header", multiple=True) +@click.option( + "--noninteractive", + is_flag=True, + help="Do not prompt, automatically paginate with delay", +) +@click.pass_context +def lib_search( # pylint: disable=unused-argument + ctx, query, json_output, page, noninteractive, **filters +): + if not query: + query = [] + if not isinstance(query, list): + query = list(query) + + for key, values in filters.items(): + for value in values: + query.append('%s:"%s"' % (key, value)) + + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg search` instead.\n", + fg="yellow", + ) + query.append("type:library") + return ctx.invoke(package_search_cmd, query=" ".join(query), page=page) + + regclient = LibraryPackageManager().get_registry_client_instance() + result = regclient.fetch_json_data( + "get", + "/v2/lib/search", + params=dict(query=" ".join(query), page=page), + x_cache_valid="1d", + ) + return click.echo(json.dumps(result)) + + +@cli.command("builtin", short_help="List built-in libraries") +@click.option("--storage", multiple=True) +@click.option("--json-output", is_flag=True) +def lib_builtin(storage, json_output): + items = LibraryPackageManager.get_builtin_libs(storage) + if json_output: + return click.echo(json.dumps(items)) + + for storage_ in items: + if not storage_["items"]: + continue + click.secho(storage_["name"], fg="green") + click.echo("*" * len(storage_["name"])) + click.echo() + + for item in sorted(storage_["items"], key=lambda i: i["name"]): + print_lib_item(item) + + return True + + +@cli.command("show", short_help="Show detailed info about a library") +@click.argument("library", metavar="[LIBRARY]") +@click.option("--json-output", is_flag=True) +@click.pass_context +def lib_show(ctx, library, json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg show` instead.\n", + fg="yellow", + ) + return ctx.invoke(package_show_cmd, pkg_type="library", spec=library) + + lm = LibraryPackageManager() + lm.set_log_level(logging.ERROR if json_output else logging.DEBUG) + lib_id = lm.reveal_registry_package_id(library) + regclient = lm.get_registry_client_instance() + lib = regclient.fetch_json_data( + "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h" + ) + return click.echo(json.dumps(lib)) + + +@cli.command("register", short_help="Deprecated") +@click.argument("config_url") +def lib_register(config_url): # pylint: disable=unused-argument + raise exception.UserSideException( + "This command is deprecated. Please use `pio pkg publish` command." + ) + + +@cli.command("stats", short_help="Library Registry Statistics") +@click.option("--json-output", is_flag=True) +def lib_stats(json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease visit " + "https://registry.platformio.org\n", + fg="yellow", + ) + return None + + regclient = LibraryPackageManager().get_registry_client_instance() + result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h") + return click.echo(json.dumps(result)) + + +def print_lib_item(item): + click.secho(item["name"], fg="cyan") + click.echo("=" * len(item["name"])) + if "id" in item: + click.secho("#ID: %d" % item["id"], bold=True) + if "description" in item or "url" in item: + click.echo(item.get("description", item.get("url", ""))) + click.echo() + + for key in ("version", "homepage", "license", "keywords"): + if key not in item or not item[key]: + continue + if isinstance(item[key], list): + click.echo("%s: %s" % (key.capitalize(), ", ".join(item[key]))) + else: + click.echo("%s: %s" % (key.capitalize(), item[key])) + + for key in ("frameworks", "platforms"): + if key not in item: + continue + click.echo( + "Compatible %s: %s" + % ( + key, + ", ".join( + [i["title"] if isinstance(i, dict) else i for i in item[key]] + ), + ) + ) + + if "authors" in item or "authornames" in item: + click.echo( + "Authors: %s" + % ", ".join( + item.get( + "authornames", [a.get("name", "") for a in item.get("authors", [])] + ) + ) + ) + + if "__src_url" in item: + click.secho("Source: %s" % item["__src_url"]) + click.echo() diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py new file mode 100644 index 0000000..2f2ae47 --- /dev/null +++ b/platformio/commands/platform.py @@ -0,0 +1,391 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os + +import click + +from platformio.exception import UserSideException +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.list import package_list_cmd +from platformio.package.commands.search import package_search_cmd +from platformio.package.commands.show import package_show_cmd +from platformio.package.commands.uninstall import package_uninstall_cmd +from platformio.package.commands.update import package_update_cmd +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageItem, PackageSpec +from platformio.package.version import get_original_version +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory + + +@click.group(short_help="Platform manager", hidden=True) +def cli(): + pass + + +@cli.command("search", short_help="Search for development platform") +@click.argument("query", required=False) +@click.option("--json-output", is_flag=True) +@click.pass_context +def platform_search(ctx, query, json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg search` instead.\n", + fg="yellow", + ) + query = query or "" + return ctx.invoke(package_search_cmd, query=f"type:platform {query}".strip()) + + platforms = [] + for platform in _get_registry_platforms(): + if query == "all": + query = "" + search_data = json.dumps(platform) + if query and query.lower() not in search_data.lower(): + continue + platforms.append( + _get_registry_platform_data( + platform["name"], with_boards=False, expose_packages=False + ) + ) + click.echo(json.dumps(platforms)) + return None + + +@cli.command("frameworks", short_help="List supported frameworks, SDKs") +@click.argument("query", required=False) +@click.option("--json-output", is_flag=True) +def platform_frameworks(query, json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease visit https://docs.platformio.org" + "/en/latest/frameworks/index.html\n", + fg="yellow", + ) + return + + regclient = PlatformPackageManager().get_registry_client_instance() + frameworks = [] + for framework in regclient.fetch_json_data( + "get", "/v2/frameworks", x_cache_valid="1d" + ): + if query == "all": + query = "" + search_data = json.dumps(framework) + if query and query.lower() not in search_data.lower(): + continue + framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"] + framework["platforms"] = [ + platform["name"] + for platform in _get_registry_platforms() + if framework["name"] in platform["frameworks"] + ] + frameworks.append(framework) + + frameworks = sorted(frameworks, key=lambda manifest: manifest["name"]) + click.echo(json.dumps(frameworks)) + + +@cli.command("list", short_help="List installed development platforms") +@click.option("--json-output", is_flag=True) +@click.pass_context +def platform_list(ctx, json_output): + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg list` instead.\n", + fg="yellow", + ) + return ctx.invoke(package_list_cmd, **{"global": True, "only_platforms": True}) + + platforms = [] + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + platforms.append( + _get_installed_platform_data(pkg, with_boards=False, expose_packages=False) + ) + + platforms = sorted(platforms, key=lambda manifest: manifest["name"]) + click.echo(json.dumps(platforms)) + return None + + +@cli.command("show", short_help="Show details about development platform") +@click.argument("platform") +@click.option("--json-output", is_flag=True) +@click.pass_context +def platform_show(ctx, platform, json_output): # pylint: disable=too-many-branches + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg show` instead.\n", + fg="yellow", + ) + return ctx.invoke(package_show_cmd, pkg_type="platform", spec=platform) + + data = _get_platform_data(platform) + if not data: + raise UnknownPlatform(platform) + return click.echo(json.dumps(data)) + + +@cli.command("install", short_help="Install new development platform") +@click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") +@click.option("--with-package", multiple=True) +@click.option("--without-package", multiple=True) +@click.option("--skip-default-package", is_flag=True) +@click.option("--with-all-packages", is_flag=True) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +@click.option( + "-f", + "--force", + is_flag=True, + help="Reinstall/redownload dev/platform and its packages if exist", +) +@click.pass_context +def platform_install( # pylint: disable=too-many-arguments + ctx, + platforms, + with_package, + without_package, + skip_default_package, + with_all_packages, + silent, + force, +): + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg install` instead.\n", + fg="yellow", + ) + ctx.invoke( + package_install_cmd, + **{ + "global": True, + "platforms": platforms, + "skip_dependencies": ( + not with_all_packages + and (with_package or without_package or skip_default_package) + ), + "silent": silent, + "force": force, + }, + ) + + +@cli.command("uninstall", short_help="Uninstall development platform") +@click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") +@click.pass_context +def platform_uninstall(ctx, platforms): + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg uninstall` instead.\n", + fg="yellow", + ) + ctx.invoke( + package_uninstall_cmd, + **{ + "global": True, + "platforms": platforms, + }, + ) + + +@cli.command("update", short_help="Update installed development platforms") +@click.argument("platforms", nargs=-1, required=False, metavar="[PLATFORM...]") +@click.option( + "-p", "--only-packages", is_flag=True, help="Update only the platform packages" +) +@click.option( + "-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead", +) +@click.option( + "--dry-run", is_flag=True, help="Do not update, only check for the new versions" +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +@click.option("--json-output", is_flag=True) +@click.pass_context +def platform_update( # pylint: disable=too-many-locals, too-many-arguments + ctx, platforms, only_check, dry_run, silent, json_output, **_ +): + only_check = dry_run or only_check + + if only_check and not json_output: + raise UserSideException( + "This command is deprecated, please use `pio pkg outdated` instead" + ) + + if not json_output: + click.secho( + "\nWARNING: This command is deprecated and will be removed in " + "the next releases. \nPlease use `pio pkg update` instead.\n", + fg="yellow", + ) + return ctx.invoke( + package_update_cmd, + **{ + "global": True, + "platforms": platforms, + "silent": silent, + }, + ) + + pm = PlatformPackageManager() + pm.set_log_level(logging.WARN if silent else logging.DEBUG) + platforms = platforms or pm.get_installed() + result = [] + for platform in platforms: + spec = None + pkg = None + if isinstance(platform, PackageItem): + pkg = platform + else: + spec = PackageSpec(platform) + pkg = pm.get_package(spec) + if not pkg: + continue + outdated = pm.outdated(pkg, spec) + if ( + not outdated.is_outdated(allow_incompatible=True) + and not PlatformFactory.new(pkg).are_outdated_packages() + ): + continue + data = _get_installed_platform_data( + pkg, with_boards=False, expose_packages=False + ) + if outdated.is_outdated(allow_incompatible=True): + data["versionLatest"] = str(outdated.latest) if outdated.latest else None + result.append(data) + click.echo(json.dumps(result)) + return True + + +# +# Helpers +# + + +def _get_registry_platforms(): + regclient = PlatformPackageManager().get_registry_client_instance() + return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d") + + +def _get_platform_data(*args, **kwargs): + try: + return _get_installed_platform_data(*args, **kwargs) + except UnknownPlatform: + return _get_registry_platform_data(*args, **kwargs) + + +def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): + p = PlatformFactory.new(platform) + data = dict( + name=p.name, + title=p.title, + description=p.description, + version=p.version, + homepage=p.homepage, + url=p.homepage, + repository=p.repository_url, + license=p.license, + forDesktop=not p.is_embedded(), + frameworks=sorted(list(p.frameworks) if p.frameworks else []), + packages=list(p.packages) if p.packages else [], + ) + + # if dump to API + # del data['version'] + # return data + + # overwrite VCS version and add extra fields + manifest = PlatformPackageManager().legacy_load_manifest( + os.path.dirname(p.manifest_path) + ) + assert manifest + for key in manifest: + if key == "version" or key.startswith("__"): + data[key] = manifest[key] + + if with_boards: + data["boards"] = [c.get_brief_data() for c in p.get_boards().values()] + + if not data["packages"] or not expose_packages: + return data + + data["packages"] = [] + installed_pkgs = { + pkg.metadata.name: p.pm.load_manifest(pkg) for pkg in p.get_installed_packages() + } + for name, options in p.packages.items(): + item = dict( + name=name, + type=p.get_package_type(name), + requirements=options.get("version"), + optional=options.get("optional") is True, + ) + if name in installed_pkgs: + for key, value in installed_pkgs[name].items(): + if key not in ("url", "version", "description"): + continue + item[key] = value + if key == "version": + item["originalVersion"] = get_original_version(value) + data["packages"].append(item) + + return data + + +def _get_registry_platform_data( # pylint: disable=unused-argument + platform, with_boards=True, expose_packages=True +): + _data = None + for p in _get_registry_platforms(): + if p["name"] == platform: + _data = p + break + + if not _data: + return None + + data = dict( + ownername=_data.get("ownername"), + name=_data["name"], + title=_data["title"], + description=_data["description"], + homepage=_data["homepage"], + repository=_data["repository"], + url=_data["url"], + license=_data["license"], + forDesktop=_data["forDesktop"], + frameworks=_data["frameworks"], + packages=_data["packages"], + versions=_data.get("versions"), + ) + + if with_boards: + data["boards"] = [ + board + for board in PlatformPackageManager().get_registered_boards() + if board["platform"] == _data["name"] + ] + + return data diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py new file mode 100644 index 0000000..695d902 --- /dev/null +++ b/platformio/commands/settings.py @@ -0,0 +1,80 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +from tabulate import tabulate + +from platformio import app +from platformio.compat import string_types + + +def format_value(raw): + if isinstance(raw, bool): + return "Yes" if raw else "No" + if isinstance(raw, string_types): + return raw + return str(raw) + + +@click.group(short_help="Manage system settings") +def cli(): + pass + + +@cli.command("get", short_help="Get existing setting/-s") +@click.argument("name", required=False) +def settings_get(name): + tabular_data = [] + for key, options in sorted(app.DEFAULT_SETTINGS.items()): + if name and name != key: + continue + raw_value = app.get_setting(key) + formatted_value = format_value(raw_value) + + if raw_value != options["value"]: + default_formatted_value = format_value(options["value"]) + formatted_value += "%s" % ( + "\n" if len(default_formatted_value) > 10 else " " + ) + formatted_value += "[%s]" % click.style( + default_formatted_value, fg="yellow" + ) + + tabular_data.append( + (click.style(key, fg="cyan"), formatted_value, options["description"]) + ) + + click.echo( + tabulate( + tabular_data, headers=["Name", "Current value [Default]", "Description"] + ) + ) + + +@cli.command("set", short_help="Set new value for the setting") +@click.argument("name") +@click.argument("value") +@click.pass_context +def settings_set(ctx, name, value): + app.set_setting(name, value) + click.secho("The new value for the setting has been set!", fg="green") + ctx.invoke(settings_get, name=name) + + +@cli.command("reset", short_help="Reset settings to default") +@click.pass_context +def settings_reset(ctx): + app.reset_settings() + click.secho("The settings have been reseted!", fg="green") + ctx.invoke(settings_get) diff --git a/platformio/commands/update.py b/platformio/commands/update.py new file mode 100644 index 0000000..a4d0e19 --- /dev/null +++ b/platformio/commands/update.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + + +@click.command( + "update", + short_help="Update installed platforms, packages and libraries", + hidden=True, +) +@click.option("--core-packages", is_flag=True, help="Update only the core packages") +@click.option( + "-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead", +) +@click.option( + "--dry-run", is_flag=True, help="Do not update, only check for the new versions" +) +def cli(*_, **__): + click.secho( + "This command is deprecated and will be removed in the next releases. \n" + "Please use `pio pkg update` instead.", + fg="yellow", + ) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py new file mode 100644 index 0000000..3fb611e --- /dev/null +++ b/platformio/commands/upgrade.py @@ -0,0 +1,153 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +import subprocess + +import click + +from platformio import VERSION, __install_requires__, __version__, app, exception +from platformio.http import fetch_remote_content +from platformio.package.manager.core import update_core_packages +from platformio.proc import get_pythonexe_path + +PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json" +DEVELOP_ZIP_URL = "https://github.com/platformio/platformio-core/archive/develop.zip" +DEVELOP_INIT_SCRIPT_URL = ( + "https://raw.githubusercontent.com/platformio/platformio-core" + "/develop/platformio/__init__.py" +) + + +@click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version") +@click.option("--dev", is_flag=True, help="Use development branch") +@click.option("--only-dependencies", is_flag=True) +@click.option("--verbose", "-v", is_flag=True) +def cli(dev, only_dependencies, verbose): + if only_dependencies: + return upgrade_pypi_dependencies(verbose) + + update_core_packages() + + if not dev and __version__ == get_latest_version(): + return click.secho( + "You're up-to-date!\nPlatformIO %s is currently the " + "newest version available." % __version__, + fg="green", + ) + + click.secho("Please wait while upgrading PlatformIO Core ...", fg="yellow") + + python_exe = get_pythonexe_path() + to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".") + pkg_spec = DEVELOP_ZIP_URL if to_develop else "platformio" + + try: + # PIO Core + subprocess.run( + [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec], + check=True, + stdout=subprocess.PIPE if not verbose else None, + ) + + # PyPI dependencies + subprocess.run( + [python_exe, "-m", "platformio", "upgrade", "--only-dependencies"], + check=False, + stdout=subprocess.PIPE, + ) + + # Check version + output = subprocess.run( + [python_exe, "-m", "platformio", "--version"], + check=True, + stdout=subprocess.PIPE, + ).stdout.decode() + assert "version" in output + actual_version = output.split("version", 1)[1].strip() + click.secho( + "PlatformIO has been successfully upgraded to %s" % actual_version, + fg="green", + ) + click.echo("Release notes: ", nl=False) + click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") + if app.get_session_var("caller_id"): + click.secho( + "Warning! Please restart IDE to affect PIO Home changes", fg="yellow" + ) + except (AssertionError, subprocess.CalledProcessError) as exc: + click.secho( + "\nWarning!!! Could not automatically upgrade the PlatformIO Core.", + fg="red", + ) + click.secho( + "Please upgrade it manually using the following command:\n", + fg="red", + ) + click.secho(f'"{python_exe}" -m pip install -U {pkg_spec}\n', fg="cyan") + raise exception.ReturnErrorCode(1) from exc + + return True + + +def upgrade_pypi_dependencies(verbose): + subprocess.run( + [ + get_pythonexe_path(), + "-m", + "pip", + "install", + "--upgrade", + "pip", + *__install_requires__, + ], + check=True, + stdout=subprocess.PIPE if not verbose else None, + ) + + +def get_latest_version(): + try: + if not str(VERSION[2]).isdigit(): + try: + return get_develop_latest_version() + except: # pylint: disable=bare-except + pass + return get_pypi_latest_version() + except Exception as exc: + raise exception.GetLatestVersionError() from exc + + +def get_develop_latest_version(): + version = None + content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL) + for line in content.split("\n"): + line = line.strip() + if not line.startswith("VERSION"): + continue + match = re.match(r"VERSION\s*=\s*\(([^\)]+)\)", line) + if not match: + continue + version = match.group(1) + for c in (" ", "'", '"'): + version = version.replace(c, "") + version = ".".join(version.split(",")) + assert version + return version + + +def get_pypi_latest_version(): + content = fetch_remote_content(PYPI_JSON_URL) + return json.loads(content)["info"]["version"] diff --git a/platformio/compat.py b/platformio/compat.py new file mode 100644 index 0000000..008dc7a --- /dev/null +++ b/platformio/compat.py @@ -0,0 +1,134 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-import,no-name-in-module + +import importlib.util +import inspect +import locale +import shlex +import sys + +from platformio.exception import UserSideException + +if sys.version_info >= (3, 7): + from asyncio import create_task as aio_create_task + from asyncio import get_running_loop as aio_get_running_loop +else: + from asyncio import ensure_future as aio_create_task + from asyncio import get_event_loop as aio_get_running_loop + + +if sys.version_info >= (3, 8): + from shlex import join as shlex_join +else: + + def shlex_join(split_command): + return " ".join(shlex.quote(arg) for arg in split_command) + + +if sys.version_info >= (3, 9): + from asyncio import to_thread as aio_to_thread +else: + from starlette.concurrency import run_in_threadpool as aio_to_thread + + +PY2 = sys.version_info[0] == 2 # DO NOT REMOVE IT. ESP8266/ESP32 depend on it +IS_CYGWIN = sys.platform.startswith("cygwin") +IS_WINDOWS = WINDOWS = sys.platform.startswith("win") +IS_MACOS = sys.platform.startswith("darwin") +MISSING = object() +string_types = (str,) + + +def is_bytes(x): + return isinstance(x, (bytes, memoryview, bytearray)) + + +def isascii(text): + if sys.version_info >= (3, 7): + return text.isascii() + for c in text or "": + if ord(c) > 127: + return False + return True + + +def is_terminal(): + try: + return sys.stdout.isatty() + except Exception: # pylint: disable=broad-except + return False + + +def ci_strings_are_equal(a, b): + if a == b: + return True + if not a or not b: + return False + return a.strip().lower() == b.strip().lower() + + +def hashlib_encode_data(data): + if is_bytes(data): + return data + if not isinstance(data, string_types): + data = str(data) + return data.encode() + + +def load_python_module(name, pathname): + spec = importlib.util.spec_from_file_location(name, pathname) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def get_filesystem_encoding(): + return sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def get_locale_encoding(): + return locale.getpreferredencoding() + + +def get_object_members(obj, ignore_private=True): + members = inspect.getmembers(obj, lambda a: not inspect.isroutine(a)) + if not ignore_private: + return members + return { + item[0]: item[1] + for item in members + if not (item[0].startswith("__") and item[0].endswith("__")) + } + + +def ensure_python3(raise_exception=True): + compatible = sys.version_info >= (3, 6) + if not raise_exception or compatible: + return compatible + raise UserSideException( + "Python 3.6 or later is required for this operation. \n" + "Please check a migration guide:\n" + "https://docs.platformio.org/en/latest/core/migration.html" + "#drop-support-for-python-2-and-3-5" + ) + + +def path_to_unicode(path): + """ + Deprecated: Compatibility with dev-platforms, + and custom device monitor filters + """ + return path diff --git a/platformio/debug/__init__.py b/platformio/debug/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/debug/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/debug/cli.py b/platformio/debug/cli.py new file mode 100644 index 0000000..5c9375a --- /dev/null +++ b/platformio/debug/cli.py @@ -0,0 +1,180 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments, too-many-locals +# pylint: disable=too-many-branches, too-many-statements + +import asyncio +import os +import signal +import subprocess + +import click + +from platformio import app, exception, fs, proc +from platformio.compat import IS_WINDOWS +from platformio.debug import helpers +from platformio.debug.config.factory import DebugConfigFactory +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.debug.process.gdb import GDBClientProcess +from platformio.exception import ReturnErrorCode +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.helpers import is_platformio_project +from platformio.project.options import ProjectOptions + + +@click.command( + "debug", + context_settings=dict(ignore_unknown_options=True), + short_help="Unified Debugger", +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), +) +@click.option( + "-c", + "--project-conf", + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), +) +@click.option("--environment", "-e", metavar="") +@click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type) +@click.option("--verbose", "-v", is_flag=True) +@click.option("--interface", type=click.Choice(["gdb"])) +@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def cli( + ctx, + project_dir, + project_conf, + environment, + load_mode, + verbose, + interface, + __unprocessed, +): + app.set_session_var("custom_project_conf", project_conf) + + # use env variables from Eclipse or CLion + for name in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): + if is_platformio_project(project_dir): + break + if os.getenv(name): + project_dir = os.getenv(name) + + with fs.cd(project_dir): + project_config = ProjectConfig.get_instance(project_conf) + project_config.validate(envs=[environment] if environment else None) + env_name = environment or helpers.get_default_debug_env(project_config) + + if not interface: + return helpers.predebug_project( + ctx, project_dir, project_config, env_name, False, verbose + ) + + configure_args = ( + ctx, + project_config, + env_name, + load_mode, + verbose, + __unprocessed, + ) + if helpers.is_gdbmi_mode(): + os.environ["PLATFORMIO_DISABLE_PROGRESSBAR"] = "true" + stream = helpers.GDBMIConsoleStream() + with proc.capture_std_streams(stream): + debug_config = _configure(*configure_args) + stream.close() + else: + debug_config = _configure(*configure_args) + + _run(project_dir, debug_config, __unprocessed) + + return None + + +def _configure(ctx, project_config, env_name, load_mode, verbose, __unprocessed): + platform = PlatformFactory.from_env(env_name, autoinstall=True) + debug_config = DebugConfigFactory.new( + platform, + project_config, + env_name, + ) + if "--version" in __unprocessed: + raise ReturnErrorCode( + subprocess.run( + [debug_config.client_executable_path, "--version"], check=True + ).returncode + ) + + try: + fs.ensure_udev_rules() + except exception.InvalidUdevRules as exc: + click.echo(str(exc)) + + rebuild_prog = False + preload = debug_config.load_cmds == ["preload"] + load_mode = load_mode or debug_config.load_mode + if load_mode == "always": + rebuild_prog = preload or not helpers.has_debug_symbols( + debug_config.program_path + ) + elif load_mode == "modified": + rebuild_prog = helpers.is_prog_obsolete( + debug_config.program_path + ) or not helpers.has_debug_symbols(debug_config.program_path) + + if not (debug_config.program_path and os.path.isfile(debug_config.program_path)): + rebuild_prog = True + + if preload or (not rebuild_prog and load_mode != "always"): + # don't load firmware through debug server + debug_config.load_cmds = [] + + if rebuild_prog: + click.echo("Preparing firmware for debugging...") + helpers.predebug_project( + ctx, os.getcwd(), project_config, env_name, preload, verbose + ) + # save SHA sum of newly created prog + if load_mode == "modified": + helpers.is_prog_obsolete(debug_config.program_path) + + if not os.path.isfile(debug_config.program_path): + raise DebugInvalidOptionsError("Program/firmware is missed") + + return debug_config + + +def _run(project_dir, debug_config, __unprocessed): + loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop() + asyncio.set_event_loop(loop) + + client = GDBClientProcess(project_dir, debug_config) + coro = client.run(__unprocessed) + try: + signal.signal(signal.SIGINT, signal.SIG_IGN) + loop.run_until_complete(coro) + if IS_WINDOWS: + client.close() + # an issue with `asyncio` executor and STIDIN, + # it cannot be closed gracefully + proc.force_exit() + finally: + client.close() + loop.close() diff --git a/platformio/debug/config/__init__.py b/platformio/debug/config/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/debug/config/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py new file mode 100644 index 0000000..cb49658 --- /dev/null +++ b/platformio/debug/config/base.py @@ -0,0 +1,247 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from platformio import fs, proc, util +from platformio.compat import string_types +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.project.config import ProjectConfig +from platformio.project.helpers import load_build_metadata +from platformio.project.options import ProjectOptions + + +class DebugConfigBase: # pylint: disable=too-many-instance-attributes + def __init__(self, platform, project_config, env_name, port=None): + self.platform = platform + self.project_config = project_config + self.env_name = env_name + self.env_options = project_config.items(env=env_name, as_dict=True) + self.build_data = self._load_build_data() + + self.tool_name = None + self.board_config = {} + self.tool_settings = {} + if "board" in self.env_options: + self.board_config = platform.board_config(self.env_options["board"]) + self.tool_name = self.board_config.get_debug_tool_name( + self.env_options.get("debug_tool") + ) + self.tool_settings = ( + self.board_config.get("debug", {}) + .get("tools", {}) + .get(self.tool_name, {}) + ) + + self._load_cmds = None + self._port = None + + self.port = port + self.server = self._configure_server() + + try: + platform.configure_debug_session(self) + except NotImplementedError: + pass + + @staticmethod + def cleanup_cmds(items): + items = ProjectConfig.parse_multi_values(items) + return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] + + @property + def program_path(self): + return self.build_data["prog_path"] + + @property + def client_executable_path(self): + return self.build_data["gdb_path"] + + @property + def load_cmds(self): + if self._load_cmds is not None: + return self._load_cmds + result = self.env_options.get("debug_load_cmds") + if not result: + result = self.tool_settings.get("load_cmds") + if not result: + # legacy + result = self.tool_settings.get("load_cmd") + if not result: + result = ProjectOptions["env.debug_load_cmds"].default + return self.cleanup_cmds(result) + + @load_cmds.setter + def load_cmds(self, cmds): + self._load_cmds = cmds + + @property + def load_mode(self): + result = self.env_options.get("debug_load_mode") + if not result: + result = self.tool_settings.get("load_mode") + return result or ProjectOptions["env.debug_load_mode"].default + + @property + def init_break(self): + missed = object() + result = self.env_options.get("debug_init_break", missed) + if result != missed: + return result + result = None + if not result: + result = self.tool_settings.get("init_break") + return result or ProjectOptions["env.debug_init_break"].default + + @property + def init_cmds(self): + return self.cleanup_cmds( + self.env_options.get("debug_init_cmds", self.tool_settings.get("init_cmds")) + ) + + @property + def extra_cmds(self): + return self.cleanup_cmds( + self.env_options.get("debug_extra_cmds") + ) + self.cleanup_cmds(self.tool_settings.get("extra_cmds")) + + @property + def port(self): + return ( + self.env_options.get("debug_port", self.tool_settings.get("port")) + or self._port + ) + + @port.setter + def port(self, value): + self._port = value + + @property + def upload_protocol(self): + return self.env_options.get( + "upload_protocol", self.board_config.get("upload", {}).get("protocol") + ) + + @property + def speed(self): + return self.env_options.get("debug_speed", self.tool_settings.get("speed")) + + @property + def server_ready_pattern(self): + return self.env_options.get( + "debug_server_ready_pattern", (self.server or {}).get("ready_pattern") + ) + + def _load_build_data(self): + data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True) + if not data: + raise DebugInvalidOptionsError("Could not load a build configuration") + return data + + def _configure_server(self): + # user disabled server in platformio.ini + if "debug_server" in self.env_options and not self.env_options.get( + "debug_server" + ): + return None + + result = None + + # specific server per a system + if isinstance(self.tool_settings.get("server", {}), list): + for item in self.tool_settings["server"][:]: + self.tool_settings["server"] = item + if util.get_systype() in item.get("system", []): + break + + # user overwrites debug server + if self.env_options.get("debug_server"): + result = { + "cwd": None, + "executable": None, + "arguments": self.env_options.get("debug_server"), + } + result["executable"] = result["arguments"][0] + result["arguments"] = result["arguments"][1:] + elif "server" in self.tool_settings: + result = self.tool_settings["server"] + server_package = result.get("package") + server_package_dir = ( + self.platform.get_package_dir(server_package) + if server_package + else None + ) + if server_package and not server_package_dir: + self.platform.install_package(server_package) + server_package_dir = self.platform.get_package_dir(server_package) + result.update( + dict( + cwd=server_package_dir if server_package else None, + executable=result.get("executable"), + arguments=[ + a.replace("$PACKAGE_DIR", server_package_dir) + if server_package_dir + else a + for a in result.get("arguments", []) + ], + ) + ) + return self.reveal_patterns(result) if result else None + + def get_init_script(self, debugger): + try: + return getattr(self, "%s_INIT_SCRIPT" % debugger.upper()) + except AttributeError as exc: + raise NotImplementedError from exc + + def reveal_patterns(self, source, recursive=True): + program_path = self.program_path or "" + patterns = { + "PLATFORMIO_CORE_DIR": self.project_config.get("platformio", "core_dir"), + "PYTHONEXE": proc.get_pythonexe_path(), + "PROJECT_DIR": os.getcwd(), + "PROG_PATH": program_path, + "PROG_DIR": os.path.dirname(program_path), + "PROG_NAME": os.path.basename(os.path.splitext(program_path)[0]), + "DEBUG_PORT": self.port, + "UPLOAD_PROTOCOL": self.upload_protocol, + "INIT_BREAK": self.init_break or "", + "LOAD_CMDS": "\n".join(self.load_cmds or []), + } + for key, value in patterns.items(): + if key.endswith(("_DIR", "_PATH")): + patterns[key] = fs.to_unix_path(value) + + def _replace(text): + for key, value in patterns.items(): + pattern = "$%s" % key + text = text.replace(pattern, value or "") + return text + + if isinstance(source, string_types): + source = _replace(source) + elif isinstance(source, (list, dict)): + items = enumerate(source) if isinstance(source, list) else source.items() + for key, value in items: + if isinstance(value, string_types): + source[key] = _replace(value) + elif isinstance(value, (list, dict)) and recursive: + source[key] = self.reveal_patterns(value, patterns) + + data = json.dumps(source) + if any(("$" + key) in data for key in patterns): + source = self.reveal_patterns(source, patterns) + + return source diff --git a/platformio/debug/config/blackmagic.py b/platformio/debug/config/blackmagic.py new file mode 100644 index 0000000..370fdb7 --- /dev/null +++ b/platformio/debug/config/blackmagic.py @@ -0,0 +1,71 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.device.finder import SerialPortFinder, is_pattern_port + + +class BlackmagicDebugConfig(DebugConfigBase): + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + set language c + set *0xE000ED0C = 0x05FA0004 + set $busy = (*0xE000ED0C & 0x4) + while ($busy) + set $busy = (*0xE000ED0C & 0x4) + end + set language auto +end + +define pio_reset_run_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +monitor swdp_scan +attach 1 +set mem inaccessible-by-default off +$LOAD_CMDS +$INIT_BREAK + +set language c +set *0xE000ED0C = 0x05FA0004 +set $busy = (*0xE000ED0C & 0x4) +while ($busy) + set $busy = (*0xE000ED0C & 0x4) +end +set language auto +""" + + @property + def port(self): + # pylint: disable=assignment-from-no-return + initial_port = DebugConfigBase.port.fget(self) + if initial_port and not is_pattern_port(initial_port): + return initial_port + port = SerialPortFinder( + board_config=self.board_config, + upload_protocol=self.tool_name, + prefer_gdb_port=True, + ).find(initial_port) + if port: + return port + raise DebugInvalidOptionsError( + "Please specify `debug_port` for the working environment" + ) + + @port.setter + def port(self, value): + self._port = value diff --git a/platformio/debug/config/factory.py b/platformio/debug/config/factory.py new file mode 100644 index 0000000..d7358b9 --- /dev/null +++ b/platformio/debug/config/factory.py @@ -0,0 +1,48 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import re + +from platformio.debug.config.generic import GenericDebugConfig +from platformio.debug.config.native import NativeDebugConfig + + +class DebugConfigFactory: + @staticmethod + def get_clsname(name): + name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) + return "%sDebugConfig" % name.lower().capitalize() + + @classmethod + def new(cls, platform, project_config, env_name): + board_config = platform.board_config( + project_config.get("env:" + env_name, "board") + ) + tool_name = ( + board_config.get_debug_tool_name( + project_config.get("env:" + env_name, "debug_tool") + ) + if board_config + else None + ) + config_cls = None + try: + mod = importlib.import_module("platformio.debug.config.%s" % tool_name) + config_cls = getattr(mod, cls.get_clsname(tool_name)) + except ModuleNotFoundError: + config_cls = ( + GenericDebugConfig if platform.is_embedded() else NativeDebugConfig + ) + return config_cls(platform, project_config, env_name) diff --git a/platformio/debug/config/generic.py b/platformio/debug/config/generic.py new file mode 100644 index 0000000..faa5a34 --- /dev/null +++ b/platformio/debug/config/generic.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class GenericDebugConfig(DebugConfigBase): + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + monitor reset halt +end + +define pio_reset_run_target + monitor reset +end + +target extended-remote $DEBUG_PORT +monitor init +$LOAD_CMDS +pio_reset_halt_target +$INIT_BREAK +""" + + def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":3333" + super().__init__(*args, **kwargs) diff --git a/platformio/debug/config/jlink.py b/platformio/debug/config/jlink.py new file mode 100644 index 0000000..a820e0a --- /dev/null +++ b/platformio/debug/config/jlink.py @@ -0,0 +1,46 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class JlinkDebugConfig(DebugConfigBase): + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + monitor reset + monitor halt +end + +define pio_reset_run_target + monitor clrbp + monitor reset + monitor go +end + +target extended-remote $DEBUG_PORT +monitor clrbp +monitor speed auto +pio_reset_halt_target +$LOAD_CMDS +$INIT_BREAK +""" + + def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":2331" + super().__init__(*args, **kwargs) + + @property + def server_ready_pattern(self): + return super().server_ready_pattern or ("Waiting for GDB connection") diff --git a/platformio/debug/config/mspdebug.py b/platformio/debug/config/mspdebug.py new file mode 100644 index 0000000..7f4d8e1 --- /dev/null +++ b/platformio/debug/config/mspdebug.py @@ -0,0 +1,36 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class MspdebugDebugConfig(DebugConfigBase): + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target +end + +define pio_reset_run_target +end + +target remote $DEBUG_PORT +monitor erase +$LOAD_CMDS +pio_reset_halt_target +$INIT_BREAK +""" + + def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":2000" + super().__init__(*args, **kwargs) diff --git a/platformio/debug/config/native.py b/platformio/debug/config/native.py new file mode 100644 index 0000000..413a9c6 --- /dev/null +++ b/platformio/debug/config/native.py @@ -0,0 +1,33 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.compat import IS_WINDOWS +from platformio.debug.config.base import DebugConfigBase + + +class NativeDebugConfig(DebugConfigBase): + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target +end + +define pio_reset_run_target +end + +define pio_restart_target +end + +$INIT_BREAK +""" + ( + "set startup-with-shell off" if not IS_WINDOWS else "" + ) diff --git a/platformio/debug/config/qemu.py b/platformio/debug/config/qemu.py new file mode 100644 index 0000000..b998c78 --- /dev/null +++ b/platformio/debug/config/qemu.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class QemuDebugConfig(DebugConfigBase): + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + monitor system_reset +end + +define pio_reset_run_target + monitor system_reset +end + +target extended-remote $DEBUG_PORT +$LOAD_CMDS +pio_reset_halt_target +$INIT_BREAK +""" + + def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":1234" + super().__init__(*args, **kwargs) diff --git a/platformio/debug/config/renode.py b/platformio/debug/config/renode.py new file mode 100644 index 0000000..f2f62d6 --- /dev/null +++ b/platformio/debug/config/renode.py @@ -0,0 +1,45 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.debug.config.base import DebugConfigBase + + +class RenodeDebugConfig(DebugConfigBase): + GDB_INIT_SCRIPT = """ +define pio_reset_halt_target + monitor machine Reset + $LOAD_CMDS + monitor start +end + +define pio_reset_run_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +$LOAD_CMDS +$INIT_BREAK +monitor start +""" + + def __init__(self, *args, **kwargs): + if "port" not in kwargs: + kwargs["port"] = ":3333" + super().__init__(*args, **kwargs) + + @property + def server_ready_pattern(self): + return super().server_ready_pattern or ( + "GDB server with all CPUs started on port" + ) diff --git a/platformio/debug/exception.py b/platformio/debug/exception.py new file mode 100644 index 0000000..5147815 --- /dev/null +++ b/platformio/debug/exception.py @@ -0,0 +1,36 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException, UserSideException + + +class DebugError(PlatformioException): + pass + + +class DebugSupportError(DebugError, UserSideException): + MESSAGE = ( + "Currently, PlatformIO does not support debugging for `{0}`.\n" + "Please request support at https://github.com/platformio/" + "platformio-core/issues \nor visit -> https://docs.platformio.org" + "/page/plus/debugging.html" + ) + + +class DebugInvalidOptionsError(DebugError, UserSideException): + pass + + +class DebugInitError(DebugError, UserSideException): + pass diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py new file mode 100644 index 0000000..8982c00 --- /dev/null +++ b/platformio/debug/helpers.py @@ -0,0 +1,160 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import sys +import time +from hashlib import sha1 +from io import BytesIO + +from platformio.cli import PlatformioCLI +from platformio.compat import is_bytes +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.run.cli import cli as cmd_run +from platformio.run.cli import print_processing_header +from platformio.test.helpers import list_test_names +from platformio.test.result import TestSuite +from platformio.test.runners.base import TestRunnerOptions +from platformio.test.runners.factory import TestRunnerFactory + + +class GDBMIConsoleStream(BytesIO): # pylint: disable=too-few-public-methods + STDOUT = sys.stdout + + def write(self, text): + self.STDOUT.write(escape_gdbmi_stream("~", text)) + self.STDOUT.flush() + + +def is_gdbmi_mode(): + return "--interpreter" in " ".join(PlatformioCLI.leftover_args) + + +def escape_gdbmi_stream(prefix, stream): + bytes_stream = False + if is_bytes(stream): + bytes_stream = True + stream = stream.decode() + + if not stream: + return b"" if bytes_stream else "" + + ends_nl = stream.endswith("\n") + stream = re.sub(r"\\+", "\\\\\\\\", stream) + stream = stream.replace('"', '\\"') + stream = stream.replace("\n", "\\n") + stream = '%s"%s"' % (prefix, stream) + if ends_nl: + stream += "\n" + + return stream.encode() if bytes_stream else stream + + +def get_default_debug_env(config): + default_envs = config.default_envs() + all_envs = config.envs() + for env in default_envs: + if config.get("env:" + env, "build_type") == "debug": + return env + for env in all_envs: + if config.get("env:" + env, "build_type") == "debug": + return env + return default_envs[0] if default_envs else all_envs[0] + + +def predebug_project( + ctx, project_dir, project_config, env_name, preload, verbose +): # pylint: disable=too-many-arguments + debug_testname = project_config.get("env:" + env_name, "debug_test") + if debug_testname: + test_names = list_test_names(project_config) + if debug_testname not in test_names: + raise DebugInvalidOptionsError( + "Unknown test name `%s`. Valid names are `%s`" + % (debug_testname, ", ".join(test_names)) + ) + print_processing_header(env_name, project_config, verbose) + test_runner = TestRunnerFactory.new( + TestSuite(env_name, debug_testname), + project_config, + TestRunnerOptions( + verbose=3 if verbose else 0, + without_building=False, + without_debugging=False, + without_uploading=not preload, + without_testing=True, + ), + ) + test_runner.start(ctx) + else: + ctx.invoke( + cmd_run, + project_dir=project_dir, + project_conf=project_config.path, + environment=[env_name], + target=["__debug"] + (["upload"] if preload else []), + verbose=verbose, + ) + + if preload: + time.sleep(5) + + +def has_debug_symbols(prog_path): + if not os.path.isfile(prog_path): + return False + matched = { + b".debug_info": False, + b".debug_abbrev": False, + b" -Og": False, + b" -g": False, + # b"__PLATFORMIO_BUILD_DEBUG__": False, + } + with open(prog_path, "rb") as fp: + last_data = b"" + while True: + data = fp.read(1024) + if not data: + break + for pattern, found in matched.items(): + if found: + continue + if pattern in last_data + data: + matched[pattern] = True + last_data = data + return all(matched.values()) + + +def is_prog_obsolete(prog_path): + prog_hash_path = prog_path + ".sha1" + if not os.path.isfile(prog_path): + return True + shasum = sha1() + with open(prog_path, "rb") as fp: + while True: + data = fp.read(1024) + if not data: + break + shasum.update(data) + new_digest = shasum.hexdigest() + old_digest = None + if os.path.isfile(prog_hash_path): + with open(prog_hash_path, encoding="utf8") as fp: + old_digest = fp.read() + if new_digest == old_digest: + return False + with open(prog_hash_path, mode="w", encoding="utf8") as fp: + fp.write(new_digest) + return True diff --git a/platformio/debug/process/__init__.py b/platformio/debug/process/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/debug/process/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py new file mode 100644 index 0000000..6ba8c70 --- /dev/null +++ b/platformio/debug/process/base.py @@ -0,0 +1,154 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import signal +import subprocess +import sys +import time + +from platformio.compat import ( + IS_WINDOWS, + aio_create_task, + aio_get_running_loop, + get_locale_encoding, +) + + +class DebugSubprocessProtocol(asyncio.SubprocessProtocol): + def __init__(self, factory): + self.factory = factory + self._is_exited = False + + def connection_made(self, transport): + self.factory.connection_made(transport) + + def pipe_data_received(self, fd, data): + pipe_to_cb = [ + self.factory.stdin_data_received, + self.factory.stdout_data_received, + self.factory.stderr_data_received, + ] + pipe_to_cb[fd](data) + + def connection_lost(self, exc): + self.process_exited() + + def process_exited(self): + if self._is_exited: + return + self.factory.process_exited() + self._is_exited = True + + +class DebugBaseProcess: + STDOUT_CHUNK_SIZE = 2048 + LOG_FILE = None + + def __init__(self): + self.transport = None + self._is_running = False + self._last_activity = 0 + self._exit_future = None + self._stdin_read_task = None + self._std_encoding = get_locale_encoding() + + async def spawn(self, *args, **kwargs): + wait_until_exit = False + if "wait_until_exit" in kwargs: + wait_until_exit = kwargs["wait_until_exit"] + del kwargs["wait_until_exit"] + for pipe in ("stdin", "stdout", "stderr"): + if pipe not in kwargs: + kwargs[pipe] = subprocess.PIPE + loop = aio_get_running_loop() + await loop.subprocess_exec( + lambda: DebugSubprocessProtocol(self), *args, **kwargs + ) + if wait_until_exit: + self._exit_future = loop.create_future() + await self._exit_future + + def is_running(self): + return self._is_running + + def connection_made(self, transport): + self._is_running = True + self.transport = transport + + def connect_stdin_pipe(self): + self._stdin_read_task = aio_create_task(self._read_stdin_pipe()) + + async def _read_stdin_pipe(self): + loop = aio_get_running_loop() + if IS_WINDOWS: + while True: + self.stdin_data_received( + await loop.run_in_executor(None, sys.stdin.buffer.readline) + ) + else: + reader = asyncio.StreamReader() + protocol = asyncio.StreamReaderProtocol(reader) + await loop.connect_read_pipe(lambda: protocol, sys.stdin) + while True: + self.stdin_data_received(await reader.readline()) + + def stdin_data_received(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: + fp.write(data) + + def stdout_data_received(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: + fp.write(data) + while data: + chunk = data[: self.STDOUT_CHUNK_SIZE] + print(chunk.decode(self._std_encoding, "replace"), end="", flush=True) + data = data[self.STDOUT_CHUNK_SIZE :] + + def stderr_data_received(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: + fp.write(data) + print( + data.decode(self._std_encoding, "replace"), + end="", + file=sys.stderr, + flush=True, + ) + + def process_exited(self): + self._is_running = False + self._last_activity = time.time() + # Allow terminating via SIGINT/CTRL+C + signal.signal(signal.SIGINT, signal.default_int_handler) + if self._stdin_read_task: + self._stdin_read_task.cancel() + self._stdin_read_task = None + if self._exit_future: + self._exit_future.set_result(True) + self._exit_future = None + + def terminate(self): + if not self.is_running() or not self.transport: + return + try: + self.transport.kill() + self.transport.close() + except: # pylint: disable=bare-except + pass diff --git a/platformio/debug/process/client.py b/platformio/debug/process/client.py new file mode 100644 index 0000000..d508bda --- /dev/null +++ b/platformio/debug/process/client.py @@ -0,0 +1,104 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import signal +import tempfile + +from platformio import fs, proc +from platformio.cache import ContentCache +from platformio.compat import IS_WINDOWS, hashlib_encode_data +from platformio.debug.process.base import DebugBaseProcess +from platformio.debug.process.server import DebugServerProcess +from platformio.project.helpers import get_project_cache_dir + + +class DebugClientProcess(DebugBaseProcess): + def __init__(self, project_dir, debug_config): + super().__init__() + self.project_dir = project_dir + self.debug_config = debug_config + + self._server_process = None + self._session_id = None + + if not os.path.isdir(get_project_cache_dir()): + os.makedirs(get_project_cache_dir()) + self.working_dir = tempfile.mkdtemp( + dir=get_project_cache_dir(), prefix=".piodebug-" + ) + + self._target_is_running = False + self._errors_buffer = b"" + + async def run(self): + session_hash = ( + self.debug_config.client_executable_path + self.debug_config.program_path + ) + self._session_id = hashlib.sha1(hashlib_encode_data(session_hash)).hexdigest() + self._kill_previous_session() + + if self.debug_config.server: + self._server_process = DebugServerProcess(self.debug_config) + self.debug_config.port = await self._server_process.run() + + def connection_made(self, transport): + super().connection_made(transport) + self._lock_session(transport.get_pid()) + # Disable SIGINT and allow GDB's Ctrl+C interrupt + signal.signal(signal.SIGINT, lambda *args, **kwargs: None) + self.connect_stdin_pipe() + + def process_exited(self): + if self._server_process: + self._server_process.terminate() + super().process_exited() + + def close(self): + self._unlock_session() + if self.working_dir and os.path.isdir(self.working_dir): + fs.rmtree(self.working_dir) + + def __del__(self): + self.close() + + def _kill_previous_session(self): + assert self._session_id + pid = None + with ContentCache() as cc: + pid = cc.get(self._session_id) + cc.delete(self._session_id) + if not pid: + return + if IS_WINDOWS: + kill = ["Taskkill", "/PID", pid, "/F"] + else: + kill = ["kill", pid] + try: + proc.exec_command(kill) + except: # pylint: disable=bare-except + pass + + def _lock_session(self, pid): + if not self._session_id: + return + with ContentCache() as cc: + cc.set(self._session_id, str(pid), "1h") + + def _unlock_session(self): + if not self._session_id: + return + with ContentCache() as cc: + cc.delete(self._session_id) diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py new file mode 100644 index 0000000..29a8ba3 --- /dev/null +++ b/platformio/debug/process/gdb.py @@ -0,0 +1,181 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import signal +import time + +from platformio import telemetry +from platformio.compat import aio_get_running_loop, is_bytes +from platformio.debug import helpers +from platformio.debug.exception import DebugInitError +from platformio.debug.process.client import DebugClientProcess + + +class GDBClientProcess(DebugClientProcess): + PIO_SRC_NAME = ".pioinit" + INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._target_is_running = False + self._errors_buffer = b"" + + async def run(self, extra_args): # pylint: disable=arguments-differ + await super().run() + + self.generate_init_script(os.path.join(self.working_dir, self.PIO_SRC_NAME)) + gdb_path = self.debug_config.client_executable_path or "gdb" + # start GDB client + args = [ + gdb_path, + "-q", + "--directory", + self.working_dir, + "--directory", + self.project_dir, + "-l", + "10", + ] + args.extend(list(extra_args or [])) + gdb_data_dir = self._get_data_dir(gdb_path) + if gdb_data_dir: + args.extend(["--data-directory", gdb_data_dir]) + args.append(self.debug_config.program_path) + + await self.spawn(*args, cwd=self.project_dir, wait_until_exit=True) + + @staticmethod + def _get_data_dir(gdb_path): + if "msp430" in gdb_path: + return None + gdb_data_dir = os.path.abspath( + os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb") + ) + return gdb_data_dir if os.path.isdir(gdb_data_dir) else None + + def generate_init_script(self, dst): + # default GDB init commands depending on debug tool + commands = self.debug_config.get_init_script("gdb").split("\n") + + if self.debug_config.init_cmds: + commands = self.debug_config.init_cmds + commands.extend(self.debug_config.extra_cmds) + + if not any("define pio_reset_run_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_run_target", + " echo Warning! Undefined pio_reset_run_target command\\n", + " monitor reset", + "end", + ] + commands + if not any("define pio_reset_halt_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_halt_target", + " echo Warning! Undefined pio_reset_halt_target command\\n", + " monitor reset halt", + "end", + ] + commands + if not any("define pio_restart_target" in cmd for cmd in commands): + commands += [ + "define pio_restart_target", + " pio_reset_halt_target", + " $INIT_BREAK", + " %s" % ("continue" if self.debug_config.init_break else "next"), + "end", + ] + + banner = [ + "echo PlatformIO Unified Debugger -> https://bit.ly/pio-debug\\n", + "echo PlatformIO: debug_tool = %s\\n" % self.debug_config.tool_name, + "echo PlatformIO: Initializing remote target...\\n", + ] + footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] + commands = banner + commands + footer + + with open(dst, mode="w", encoding="utf8") as fp: + fp.write("\n".join(self.debug_config.reveal_patterns(commands))) + + def stdin_data_received(self, data): + super().stdin_data_received(data) + if b"-exec-run" in data: + if self._target_is_running: + token, _ = data.split(b"-", 1) + self.stdout_data_received(token + b"^running\n") + return + if self.debug_config.platform.is_embedded(): + data = data.replace(b"-exec-run", b"-exec-continue") + + if b"-exec-continue" in data: + self._target_is_running = True + if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): + # Allow terminating via SIGINT/CTRL+C + signal.signal(signal.SIGINT, signal.default_int_handler) + self.transport.get_pipe_transport(0).write(b"pio_reset_run_target\n") + self.transport.get_pipe_transport(0).write(data) + + def stdout_data_received(self, data): + super().stdout_data_received(data) + self._handle_error(data) + # go to init break automatically + if self.INIT_COMPLETED_BANNER.encode() in data: + telemetry.log_debug_started(self.debug_config) + self._auto_exec_continue() + + def console_log(self, msg): + if helpers.is_gdbmi_mode(): + msg = helpers.escape_gdbmi_stream("~", msg) + self.stdout_data_received(msg if is_bytes(msg) else msg.encode()) + + def _auto_exec_continue(self): + auto_exec_delay = 0.5 # in seconds + if self._last_activity > (time.time() - auto_exec_delay): + aio_get_running_loop().call_later(0.1, self._auto_exec_continue) + return + + if not self.debug_config.init_break or self._target_is_running: + return + + self.console_log( + "PlatformIO: Resume the execution to `debug_init_break = %s`\n" + % self.debug_config.init_break + ) + self.console_log( + "PlatformIO: More configuration options -> https://bit.ly/pio-debug\n" + ) + if self.debug_config.platform.is_embedded(): + self.transport.get_pipe_transport(0).write( + b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n" + ) + else: + self.transport.get_pipe_transport(0).write( + b"0-exec-run\n" if helpers.is_gdbmi_mode() else b"run\n" + ) + self._target_is_running = True + + def stderr_data_received(self, data): + super().stderr_data_received(data) + self._handle_error(data) + + def _handle_error(self, data): + self._errors_buffer = (self._errors_buffer + data)[-8192:] # keep last 8 KBytes + if not ( + self.PIO_SRC_NAME.encode() in self._errors_buffer + and b"Error in sourced" in self._errors_buffer + ): + return + telemetry.log_debug_exception( + DebugInitError(self._errors_buffer.decode()), self.debug_config + ) + self.transport.close() diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py new file mode 100644 index 0000000..693dfa4 --- /dev/null +++ b/platformio/debug/process/server.py @@ -0,0 +1,147 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import os +import re +import time + +from platformio import fs +from platformio.compat import IS_MACOS, IS_WINDOWS +from platformio.debug.exception import DebugInvalidOptionsError +from platformio.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode +from platformio.debug.process.base import DebugBaseProcess +from platformio.proc import where_is_program + + +class DebugServerProcess(DebugBaseProcess): + STD_BUFFER_SIZE = 1024 + + def __init__(self, debug_config): + super().__init__() + self.debug_config = debug_config + self._ready = False + self._std_buffer = {"out": b"", "err": b""} + + async def run(self): # pylint: disable=too-many-branches + server = self.debug_config.server + if not server: + return None + server_executable = server["executable"] + if not server_executable: + return None + if server["cwd"]: + server_executable = os.path.join(server["cwd"], server_executable) + if ( + IS_WINDOWS + and not server_executable.endswith(".exe") + and os.path.isfile(server_executable + ".exe") + ): + server_executable = server_executable + ".exe" + + if not os.path.isfile(server_executable): + server_executable = where_is_program(server_executable) + if not os.path.isfile(server_executable): + raise DebugInvalidOptionsError( + "Could not launch Debug Server '%s'. Please check that it " + "is installed and is included in a system PATH\n" + "See https://docs.platformio.org/page/plus/debugging.html" + % server_executable + ) + + openocd_pipe_allowed = all( + [ + not self.debug_config.env_options.get("debug_port"), + "gdb" in self.debug_config.client_executable_path, + "openocd" in server_executable, + ] + ) + if openocd_pipe_allowed: + args = [] + if server["cwd"]: + args.extend(["-s", server["cwd"]]) + args.extend( + ["-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"] + ) + args.extend(server["arguments"]) + str_args = " ".join( + [arg if arg.startswith("-") else '"%s"' % arg for arg in args] + ) + return fs.to_unix_path('| "%s" %s' % (server_executable, str_args)) + + env = os.environ.copy() + # prepend server "lib" folder to LD path + if ( + not IS_WINDOWS + and server["cwd"] + and os.path.isdir(os.path.join(server["cwd"], "lib")) + ): + ld_key = "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH" + env[ld_key] = os.path.join(server["cwd"], "lib") + if os.environ.get(ld_key): + env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) + # prepend BIN to PATH + if server["cwd"] and os.path.isdir(os.path.join(server["cwd"], "bin")): + env["PATH"] = "%s%s%s" % ( + os.path.join(server["cwd"], "bin"), + os.pathsep, + os.environ.get("PATH", os.environ.get("Path", "")), + ) + + await self.spawn( + *([server_executable] + server["arguments"]), cwd=server["cwd"], env=env + ) + await self._wait_until_ready() + + return self.debug_config.port + + async def _wait_until_ready(self): + ready_pattern = self.debug_config.server_ready_pattern + timeout = 60 if ready_pattern else 10 + elapsed = 0 + delay = 0.5 + auto_ready_delay = 0.5 + while not self._ready and self.is_running() and elapsed < timeout: + await asyncio.sleep(delay) + if not ready_pattern: + self._ready = self._last_activity < (time.time() - auto_ready_delay) + elapsed += delay + + def _check_ready_by_pattern(self, data): + if self._ready: + return self._ready + ready_pattern = self.debug_config.server_ready_pattern + if ready_pattern: + if ready_pattern.startswith("^"): + self._ready = re.match( + ready_pattern, + data.decode("utf-8", "ignore"), + ) + else: + self._ready = ready_pattern.encode() in data + return self._ready + + def stdout_data_received(self, data): + super().stdout_data_received( + escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data + ) + self._std_buffer["out"] += data + self._check_ready_by_pattern(self._std_buffer["out"]) + self._std_buffer["out"] = self._std_buffer["out"][-1 * self.STD_BUFFER_SIZE :] + + def stderr_data_received(self, data): + super().stderr_data_received(data) + self._std_buffer["err"] += data + self._check_ready_by_pattern(self._std_buffer["err"]) + self._std_buffer["err"] = self._std_buffer["err"][-1 * self.STD_BUFFER_SIZE :] diff --git a/platformio/device/__init__.py b/platformio/device/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/device/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/device/cli.py b/platformio/device/cli.py new file mode 100644 index 0000000..13dec79 --- /dev/null +++ b/platformio/device/cli.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.device.list.command import device_list_cmd +from platformio.device.monitor.command import device_monitor_cmd + + +@click.group( + "device", + commands=[ + device_list_cmd, + device_monitor_cmd, + ], + short_help="Device manager & Serial/Socket monitor", +) +def cli(): + pass diff --git a/platformio/device/finder.py b/platformio/device/finder.py new file mode 100644 index 0000000..621adae --- /dev/null +++ b/platformio/device/finder.py @@ -0,0 +1,257 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from fnmatch import fnmatch +from functools import lru_cache + +import click +import serial + +from platformio.compat import IS_WINDOWS +from platformio.device.list.util import list_logical_devices, list_serial_ports +from platformio.fs import get_platformio_udev_rules_path +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory +from platformio.util import retry + +BLACK_MAGIC_HWIDS = [ + "1D50:6018", +] + + +def parse_udev_rules_hwids(path): + result = [] + with open(path, mode="r", encoding="utf8") as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + attrs = {} + for attr in line.split(","): + attr = attr.replace("==", "=").replace('"', "").strip() + if "=" not in attr: + continue + name, value = attr.split("=", 1) + attrs[name] = value + hwid = "%s:%s" % ( + attrs.get("ATTRS{idVendor}", "*"), + attrs.get("ATTRS{idProduct}", "*"), + ) + if hwid != "*:*": + result.append(hwid.upper()) + return result + + +def is_pattern_port(port): + if not port: + return False + return set(["*", "?", "[", "]"]) & set(port) + + +def find_mbed_disk(initial_port): + msdlabels = ("mbed", "nucleo", "frdm", "microbit") + for item in list_logical_devices(): + if item["path"].startswith("/net"): + continue + if ( + initial_port + and is_pattern_port(initial_port) + and not fnmatch(item["path"], initial_port) + ): + continue + mbed_pages = [os.path.join(item["path"], n) for n in ("mbed.htm", "mbed.html")] + if any(os.path.isfile(p) for p in mbed_pages): + return item["path"] + if item["name"] and any(l in item["name"].lower() for l in msdlabels): + return item["path"] + return None + + +def is_serial_port_ready(port, timeout=1): + try: + serial.Serial(port, timeout=timeout).close() + return True + except: # pylint: disable=bare-except + pass + return False + + +class SerialPortFinder: + def __init__( # pylint: disable=too-many-arguments + self, + board_config=None, + upload_protocol=None, + ensure_ready=False, + prefer_gdb_port=False, + timeout=2, + verbose=False, + ): + self.board_config = board_config + self.upload_protocol = upload_protocol + self.ensure_ready = ensure_ready + self.prefer_gdb_port = prefer_gdb_port + self.timeout = timeout + self.verbose = verbose + + @staticmethod + def normalize_board_hwid(value): + if isinstance(value, (list, tuple)): + value = ("%s:%s" % (value[0], value[1])).replace("0x", "") + return value.upper() + + @staticmethod + def match_serial_port(pattern): + for item in list_serial_ports(): + if fnmatch(item["port"], pattern): + return item["port"] + return None + + @staticmethod + def match_device_hwid(patterns): + if not patterns: + return None + for item in list_serial_ports(as_objects=True): + if not item.vid or not item.pid: + continue + hwid = "{:04X}:{:04X}".format(item.vid, item.pid) + for pattern in patterns: + if fnmatch(hwid, pattern): + return item + return None + + def find(self, initial_port=None): + if initial_port: + if not is_pattern_port(initial_port): + return initial_port + return self.match_serial_port(initial_port) + + if self.upload_protocol and self.upload_protocol.startswith("blackmagic"): + return self._find_blackmagic_port() + + device = None + if self.board_config and self.board_config.get("build.hwids", []): + device = self._find_board_device() + if not device: + device = self._find_known_device() + if device: + return self._reveal_device_port(device) + + # pick the best PID:VID USB device + port = best_port = None + for item in list_serial_ports(): + if self.ensure_ready and not is_serial_port_ready(item["port"]): + continue + port = item["port"] + if "VID:PID" in item["hwid"]: + best_port = port + return best_port or port + + def _reveal_device_port(self, device): + candidates = [] + for item in list_serial_ports(as_objects=True): + if item.vid == device.vid and item.pid == device.pid: + candidates.append(item) + if len(candidates) <= 1: + return device.device + for item in candidates: + if ("GDB" if self.prefer_gdb_port else "UART") in item.description: + return item.device + candidates = sorted(candidates, key=lambda item: item.device) + # first port is GDB? BlackMagic, ESP-Prog + return candidates[0 if self.prefer_gdb_port else -1].device + + def _find_blackmagic_port(self): + device = self.match_device_hwid(BLACK_MAGIC_HWIDS) + if not device: + return None + port = self._reveal_device_port(device) + if IS_WINDOWS and port.startswith("COM") and len(port) > 4: + return "\\\\.\\%s" % port + return port + + def _find_board_device(self): + hwids = [ + self.normalize_board_hwid(hwid) + for hwid in self.board_config.get("build.hwids", []) + ] + try: + + @retry(timeout=self.timeout) + def wrapper(): + device = self.match_device_hwid(hwids) + if device: + return device + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass + + if self.verbose: + click.secho( + "TimeoutError: Could not automatically find serial port " + "for the `%s` board based on the declared HWIDs=%s" + % (self.board_config.get("name", "unknown"), hwids), + fg="yellow", + err=True, + ) + + return None + + def _find_known_device(self): + hwids = list(BLACK_MAGIC_HWIDS) + + # load from UDEV rules + udev_rules_path = get_platformio_udev_rules_path() + if os.path.isfile(udev_rules_path): + hwids.extend(parse_udev_rules_hwids(udev_rules_path)) + + @lru_cache(maxsize=1) + def _fetch_hwids_from_platforms(): + """load from installed dev-platforms""" + result = [] + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for board_config in p.get_boards().values(): + for board_hwid in board_config.get("build.hwids", []): + board_hwid = self.normalize_board_hwid(board_hwid) + if board_hwid not in result: + result.append(board_hwid) + return result + + try: + + @retry(timeout=self.timeout) + def wrapper(): + device = self.match_device_hwid(hwids) + if not device: + device = self.match_device_hwid(_fetch_hwids_from_platforms()) + if device: + return device + raise retry.RetryNextException() + + return wrapper() + except retry.RetryStopException: + pass + + if self.verbose: + click.secho( + "TimeoutError: Could not automatically find serial port " + "based on the known UART bridges", + fg="yellow", + err=True, + ) + + return None diff --git a/platformio/device/list/__init__.py b/platformio/device/list/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/device/list/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/device/list/command.py b/platformio/device/list/command.py new file mode 100644 index 0000000..078371c --- /dev/null +++ b/platformio/device/list/command.py @@ -0,0 +1,99 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click + +from platformio.device.list.util import ( + list_logical_devices, + list_mdns_services, + list_serial_ports, +) + + +@click.command("list", short_help="List devices") +@click.option("--serial", is_flag=True, help="List serial ports, default") +@click.option("--logical", is_flag=True, help="List logical devices") +@click.option("--mdns", is_flag=True, help="List multicast DNS services") +@click.option("--json-output", is_flag=True) +def device_list_cmd( # pylint: disable=too-many-branches + serial, logical, mdns, json_output +): + if not logical and not mdns: + serial = True + data = {} + if serial: + data["serial"] = list_serial_ports() + if logical: + data["logical"] = list_logical_devices() + if mdns: + data["mdns"] = list_mdns_services() + + single_key = list(data)[0] if len(list(data)) == 1 else None + + if json_output: + return click.echo(json.dumps(data[single_key] if single_key else data)) + + titles = { + "serial": "Serial Ports", + "logical": "Logical Devices", + "mdns": "Multicast DNS Services", + } + + for key, value in data.items(): + if not single_key: + click.secho(titles[key], bold=True) + click.echo("=" * len(titles[key])) + + if key == "serial": + for item in value: + click.secho(item["port"], fg="cyan") + click.echo("-" * len(item["port"])) + click.echo("Hardware ID: %s" % item["hwid"]) + click.echo("Description: %s" % item["description"]) + click.echo("") + + if key == "logical": + for item in value: + click.secho(item["path"], fg="cyan") + click.echo("-" * len(item["path"])) + click.echo("Name: %s" % item["name"]) + click.echo("") + + if key == "mdns": + for item in value: + click.secho(item["name"], fg="cyan") + click.echo("-" * len(item["name"])) + click.echo("Type: %s" % item["type"]) + click.echo("IP: %s" % item["ip"]) + click.echo("Port: %s" % item["port"]) + if item["properties"]: + click.echo( + "Properties: %s" + % ( + "; ".join( + [ + "%s=%s" % (k, v) + for k, v in item["properties"].items() + ] + ) + ) + ) + click.echo("") + + if single_key: + click.echo("") + + return True diff --git a/platformio/device/list/util.py b/platformio/device/list/util.py new file mode 100644 index 0000000..3589ee2 --- /dev/null +++ b/platformio/device/list/util.py @@ -0,0 +1,165 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import time +from glob import glob + +from platformio import __version__, exception, proc +from platformio.compat import IS_MACOS, IS_WINDOWS + + +def list_serial_ports(filter_hwid=False, as_objects=False): + try: + # pylint: disable=import-outside-toplevel + from serial.tools.list_ports import comports + except ImportError as exc: + raise exception.GetSerialPortsError(os.name) from exc + + if as_objects: + return comports() + + result = [] + for p, d, h in comports(): + if not p: + continue + if not filter_hwid or "VID:PID" in h: + result.append({"port": p, "description": d, "hwid": h}) + + if filter_hwid: + return result + + # fix for PySerial + if not result and IS_MACOS: + for p in glob("/dev/tty.*"): + result.append({"port": p, "description": "n/a", "hwid": "n/a"}) + return result + + +def list_logical_devices(): + items = [] + if IS_WINDOWS: + try: + result = proc.exec_command( + ["wmic", "logicaldisk", "get", "name,VolumeName"] + ).get("out", "") + devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") + for line in result.split("\n"): + match = devicenamere.match(line.strip()) + if not match: + continue + items.append({"path": match.group(1) + "\\", "name": match.group(2)}) + return items + except WindowsError: # pylint: disable=undefined-variable + pass + # try "fsutil" + result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") + for device in re.findall(r"[A-Z]:\\", result): + items.append({"path": device, "name": None}) + return items + + result = proc.exec_command(["df"]).get("out") + devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) + for line in result.split("\n"): + match = devicenamere.match(line.strip()) + if not match: + continue + items.append({"path": match.group(1), "name": os.path.basename(match.group(1))}) + return items + + +def list_mdns_services(): + try: + import zeroconf # pylint: disable=import-outside-toplevel + except ImportError: + result = proc.exec_command( + [proc.get_pythonexe_path(), "-m", "pip", "install", "zeroconf"] + ) + if result.get("returncode") != 0: + print(result.get("err")) + import zeroconf # pylint: disable=import-outside-toplevel + + class mDNSListener: + def __init__(self): + self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All) + self._found_types = [] + self._found_services = [] + + def __enter__(self): + zeroconf.ServiceBrowser( + self._zc, + [ + "_http._tcp.local.", + "_hap._tcp.local.", + "_services._dns-sd._udp.local.", + ], + self, + ) + return self + + def __exit__(self, etype, value, traceback): + self._zc.close() + + def add_service(self, zc, type_, name): + try: + assert zeroconf.service_type_name(name) + assert str(name) + except (AssertionError, UnicodeError, zeroconf.BadTypeInNameException): + return + if name not in self._found_types: + self._found_types.append(name) + zeroconf.ServiceBrowser(self._zc, name, self) + if type_ in self._found_types: + s = zc.get_service_info(type_, name) + if s: + self._found_services.append(s) + + def remove_service(self, zc, type_, name): + pass + + def update_service(self, zc, type_, name): + pass + + def get_services(self): + return self._found_services + + items = [] + with mDNSListener() as mdns: + time.sleep(3) + for service in mdns.get_services(): + properties = None + if service.properties: + try: + properties = { + k.decode("utf8"): v.decode("utf8") + if isinstance(v, bytes) + else v + for k, v in service.properties.items() + } + json.dumps(properties) + except UnicodeDecodeError: + properties = None + + items.append( + { + "type": service.type, + "name": service.name, + "ip": ", ".join(service.parsed_addresses()), + "port": service.port, + "properties": properties, + } + ) + return items diff --git a/platformio/device/monitor/__init__.py b/platformio/device/monitor/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/device/monitor/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py new file mode 100644 index 0000000..3d3190f --- /dev/null +++ b/platformio/device/monitor/command.py @@ -0,0 +1,177 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +import click + +from platformio import exception, fs +from platformio.device.finder import SerialPortFinder +from platformio.device.monitor.filters.base import register_filters +from platformio.device.monitor.terminal import get_available_filters, start_terminal +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError +from platformio.project.options import ProjectOptions + + +@click.command("monitor", short_help="Monitor device (Serial/Socket)") +@click.option("--port", "-p", help="Port, a number or a device name") +@click.option( + "-b", + "--baud", + type=ProjectOptions["env.monitor_speed"].type, + help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, +) +@click.option( + "--parity", + type=ProjectOptions["env.monitor_parity"].type, + help="Enable parity checking [default=%s]" + % ProjectOptions["env.monitor_parity"].default, +) +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") +@click.option("--xonxoff", is_flag=True, help="Enable software flow control") +@click.option( + "--rts", + type=ProjectOptions["env.monitor_rts"].type, + help="Set initial RTS line state", +) +@click.option( + "--dtr", + type=ProjectOptions["env.monitor_dtr"].type, + help="Set initial DTR line state", +) +@click.option("--echo", is_flag=True, help="Enable local echo") +@click.option( + "--encoding", + help=( + "Set the encoding for the serial port " + "(e.g. hexlify, Latin1, UTF-8) [default=%s]" + % ProjectOptions["env.monitor_encoding"].default + ), +) +@click.option( + "-f", + "--filter", + "filters", + multiple=True, + help="Apply filters/text transformations", +) +@click.option( + "--eol", + type=ProjectOptions["env.monitor_eol"].type, + help="End of line mode [default=%s]" % ProjectOptions["env.monitor_eol"].default, +) +@click.option("--raw", is_flag=True, help=ProjectOptions["env.monitor_raw"].description) +@click.option( + "--exit-char", + type=int, + default=3, + show_default=True, + help="ASCII code of special character that is used to exit " + "the application [default=3 (Ctrl+C)]", +) +@click.option( + "--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control terminal (menu) [default=20 (DEC)]", +) +@click.option( + "--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages", +) +@click.option( + "--no-reconnect", + is_flag=True, + help="Disable automatic reconnection if the established connection fails", +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option( + "-e", + "--environment", + help="Load configuration from `platformio.ini` and the specified environment", +) +def device_monitor_cmd(**options): + with fs.cd(options["project_dir"]): + platform = None + project_options = {} + try: + project_options = get_project_options(options["environment"]) + if "platform" in project_options: + platform = PlatformFactory.new(project_options["platform"]) + except NotPlatformIOProjectError: + pass + + options = apply_project_monitor_options(options, project_options) + register_filters(platform=platform, options=options) + options["port"] = SerialPortFinder( + board_config=platform.board_config(project_options.get("board")) + if platform and project_options.get("board") + else None, + upload_protocol=project_options.get("upload_protocol"), + ensure_ready=True, + ).find(initial_port=options["port"]) + + if options["menu_char"] == options["exit_char"]: + raise exception.UserSideException( + "--exit-char can not be the same as --menu-char" + ) + + # check for unknown filters + if options["filters"]: + known_filters = set(get_available_filters()) + unknown_filters = set(options["filters"]) - known_filters + if unknown_filters: + options["filters"] = list(known_filters & set(options["filters"])) + click.secho( + ("Warning! Skipping unknown filters `%s`. Known filters are `%s`") + % (", ".join(unknown_filters), ", ".join(sorted(known_filters))), + fg="yellow", + ) + + start_terminal(options) + + +def get_project_options(environment=None): + config = ProjectConfig.get_instance() + config.validate(envs=[environment] if environment else None) + environment = environment or config.get_default_env() + return config.items(env=environment, as_dict=True) + + +def apply_project_monitor_options(initial_options, project_options): + for option_meta in ProjectOptions.values(): + if option_meta.group != "monitor": + continue + cli_key = option_meta.name.split("_", 1)[1] + if cli_key == "speed": + cli_key = "baud" + # value set from CLI, skip overriding + if initial_options[cli_key] not in (None, (), []) and ( + option_meta.type != click.BOOL or f"--{cli_key}" in sys.argv[1:] + ): + continue + initial_options[cli_key] = project_options.get( + option_meta.name, option_meta.default + ) + return initial_options diff --git a/platformio/device/monitor/filters/__init__.py b/platformio/device/monitor/filters/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/device/monitor/filters/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/device/monitor/filters/base.py b/platformio/device/monitor/filters/base.py new file mode 100644 index 0000000..e773ed6 --- /dev/null +++ b/platformio/device/monitor/filters/base.py @@ -0,0 +1,100 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import os + +from serial.tools import miniterm + +from platformio.compat import get_object_members, load_python_module +from platformio.package.manager.tool import ToolPackageManager +from platformio.project.config import ProjectConfig + + +class DeviceMonitorFilterBase(miniterm.Transform): + def __init__(self, options=None): + """Called by PlatformIO to pass context""" + miniterm.Transform.__init__(self) + + self.options = options or {} + self.project_dir = self.options.get("project_dir") + self.environment = self.options.get("environment") + + self.config = ProjectConfig.get_instance() + if not self.environment: + default_envs = self.config.default_envs() + if default_envs: + self.environment = default_envs[0] + elif self.config.envs(): + self.environment = self.config.envs()[0] + + def __call__(self): + """Called by the miniterm library when the filter is actually used""" + return self + + @property + def NAME(self): + raise NotImplementedError("Please declare NAME attribute for the filter class") + + +def register_filters(platform=None, options=None): + # project filters + load_monitor_filters( + ProjectConfig.get_instance().get("platformio", "monitor_dir"), + prefix="filter_", + options=options, + ) + # platform filters + if platform: + load_monitor_filters( + os.path.join(platform.get_dir(), "monitor"), + prefix="filter_", + options=options, + ) + # load package filters + pm = ToolPackageManager() + for pkg in pm.get_installed(): + load_monitor_filters( + os.path.join(pkg.path, "monitor"), prefix="filter_", options=options + ) + # default filters + load_monitor_filters(os.path.dirname(__file__), options=options) + + +def load_monitor_filters(monitor_dir, prefix=None, options=None): + if not os.path.isdir(monitor_dir): + return + for name in os.listdir(monitor_dir): + if (prefix and not name.startswith(prefix)) or not name.endswith(".py"): + continue + path = os.path.join(monitor_dir, name) + if not os.path.isfile(path): + continue + load_monitor_filter(path, options) + + +def load_monitor_filter(path, options=None): + name = os.path.basename(path) + name = name[: name.find(".")] + module = load_python_module("platformio.device.monitor.filters.%s" % name, path) + for cls in get_object_members(module).values(): + if ( + not inspect.isclass(cls) + or not issubclass(cls, DeviceMonitorFilterBase) + or cls == DeviceMonitorFilterBase + ): + continue + obj = cls(options) + miniterm.TRANSFORMATIONS[obj.NAME] = obj + return True diff --git a/platformio/device/monitor/filters/hexlify.py b/platformio/device/monitor/filters/hexlify.py new file mode 100644 index 0000000..28e83bf --- /dev/null +++ b/platformio/device/monitor/filters/hexlify.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import serial + +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase + + +class Hexlify(DeviceMonitorFilterBase): + NAME = "hexlify" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._counter = 0 + + def rx(self, text): + result = "" + for b in serial.iterbytes(text): + if (self._counter % 16) == 0: + result += "\n{:04X} | ".format(self._counter) + asciicode = ord(b) + if asciicode <= 255: + result += "{:02X} ".format(asciicode) + else: + result += "?? " + self._counter += 1 + return result diff --git a/platformio/device/monitor/filters/log2file.py b/platformio/device/monitor/filters/log2file.py new file mode 100644 index 0000000..8e00d29 --- /dev/null +++ b/platformio/device/monitor/filters/log2file.py @@ -0,0 +1,47 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os +from datetime import datetime + +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase + + +class LogToFile(DeviceMonitorFilterBase): + NAME = "log2file" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._log_fp = None + + def __call__(self): + if not os.path.isdir("logs"): + os.makedirs("logs") + log_file_name = os.path.join( + "logs", "device-monitor-%s.log" % datetime.now().strftime("%y%m%d-%H%M%S") + ) + print("--- Logging an output to %s" % os.path.abspath(log_file_name)) + # pylint: disable=consider-using-with + self._log_fp = io.open(log_file_name, "w", encoding="utf-8") + return self + + def __del__(self): + if self._log_fp: + self._log_fp.close() + + def rx(self, text): + self._log_fp.write(text) + self._log_fp.flush() + return text diff --git a/platformio/device/monitor/filters/send_on_enter.py b/platformio/device/monitor/filters/send_on_enter.py new file mode 100644 index 0000000..c28888c --- /dev/null +++ b/platformio/device/monitor/filters/send_on_enter.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase + + +class SendOnEnter(DeviceMonitorFilterBase): + NAME = "send_on_enter" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._buffer = "" + + if self.options.get("eol") == "CR": + self._eol = "\r" + elif self.options.get("eol") == "LF": + self._eol = "\n" + else: + self._eol = "\r\n" + + def tx(self, text): + self._buffer += text + if self._buffer.endswith(self._eol): + text = self._buffer + self._buffer = "" + return text + return "" diff --git a/platformio/device/monitor/filters/time.py b/platformio/device/monitor/filters/time.py new file mode 100644 index 0000000..cde4e77 --- /dev/null +++ b/platformio/device/monitor/filters/time.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase + + +class Timestamp(DeviceMonitorFilterBase): + NAME = "time" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._line_started = False + + def rx(self, text): + if self._line_started and "\n" not in text: + return text + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + if not self._line_started: + self._line_started = True + text = "%s > %s" % (timestamp, text) + if text.endswith("\n"): + self._line_started = False + return text[:-1].replace("\n", "\n%s > " % timestamp) + "\n" + return text.replace("\n", "\n%s > " % timestamp) diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py new file mode 100644 index 0000000..7b2a9b9 --- /dev/null +++ b/platformio/device/monitor/terminal.py @@ -0,0 +1,188 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import signal +import sys +import threading + +import click +import serial +from serial.tools import miniterm + +from platformio.exception import UserSideException + + +class Terminal(miniterm.Miniterm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.pio_unexpected_exception = None + + def reader(self): + try: + super().reader() + except Exception as exc: # pylint: disable=broad-except + self.pio_unexpected_exception = exc + + def writer(self): + try: + super().writer() + except Exception as exc: # pylint: disable=broad-except + self.pio_unexpected_exception = exc + + +def get_available_filters(): + return sorted(miniterm.TRANSFORMATIONS.keys()) + + +def start_terminal(options): + retries = 0 + is_port_valid = False + while True: + term = None + try: + term = new_terminal(options) + is_port_valid = True + options["port"] = term.serial.name + if retries: + click.echo("\t Connected!", err=True) + elif not options["quiet"]: + print_terminal_settings(term) + retries = 0 # reset + term.start() + try: + term.join(True) + except KeyboardInterrupt: + pass + term.join() + + # cleanup + term.console.cleanup() + + # restore original standard streams + sys.stdin = sys.__stdin__ + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + + term.close() + + if term.pio_unexpected_exception: + click.secho( + "Disconnected (%s)" % term.pio_unexpected_exception, + fg="red", + err=True, + ) + if not options["no_reconnect"]: + raise UserSideException(term.pio_unexpected_exception) + + return + except UserSideException as exc: + if not is_port_valid: + raise exc + if not retries: + click.echo("Reconnecting to %s " % options["port"], err=True, nl=False) + signal.signal(signal.SIGINT, signal.SIG_DFL) + else: + click.echo(".", err=True, nl=False) + retries += 1 + threading.Event().wait(retries / 2) + + +def new_terminal(options): + term = Terminal( + new_serial_instance(options), + echo=options["echo"], + eol=options["eol"].lower(), + filters=list(reversed(options["filters"] or ["default"])), + ) + term.exit_character = chr(options["exit_char"]) + term.menu_character = chr(options["menu_char"]) + term.raw = options["raw"] + term.set_rx_encoding(options["encoding"]) + term.set_tx_encoding(options["encoding"]) + return term + + +def print_terminal_settings(terminal): + click.echo( + "--- Terminal on {p.name} | " + "{p.baudrate} {p.bytesize}-{p.parity}-{p.stopbits}".format(p=terminal.serial) + ) + click.echo( + "--- Available filters and text transformations: %s" + % ", ".join(get_available_filters()) + ) + click.echo("--- More details at https://bit.ly/pio-monitor-filters") + click.echo( + "--- Quit: {} | Menu: {} | Help: {} followed by {}".format( + miniterm.key_description(terminal.exit_character), + miniterm.key_description(terminal.menu_character), + miniterm.key_description(terminal.menu_character), + miniterm.key_description("\x08"), + ) + ) + + +def new_serial_instance(options): # pylint: disable=too-many-branches + serial_instance = None + port = options["port"] + while serial_instance is None: + # no port given on command line -> ask user now + if port is None or port == "-": + try: + port = miniterm.ask_for_port() + except KeyboardInterrupt as exc: + click.echo("", err=True) + raise UserSideException("User aborted and port is not given") from exc + if not port: + raise UserSideException("Port is not given") + try: + serial_instance = serial.serial_for_url( + port, + options["baud"], + parity=options["parity"], + rtscts=options["rtscts"], + xonxoff=options["xonxoff"], + do_not_open=True, + ) + + if not hasattr(serial_instance, "cancel_read"): + # enable timeout for alive flag polling if cancel_read is not available + serial_instance.timeout = 1 + + if options["dtr"] is not None: + if not options["quiet"]: + click.echo( + "--- forcing DTR {}".format( + "active" if options["dtr"] else "inactive" + ) + ) + serial_instance.dtr = options["dtr"] + + if options["rts"] is not None: + if not options["quiet"]: + click.echo( + "--- forcing RTS {}".format( + "active" if options["rts"] else "inactive" + ) + ) + serial_instance.rts = options["rts"] + + if isinstance(serial_instance, serial.Serial): + serial_instance.exclusive = True + + serial_instance.open() + except serial.SerialException as exc: + raise UserSideException(exc) from exc + + return serial_instance diff --git a/platformio/exception.py b/platformio/exception.py new file mode 100644 index 0000000..80ffb49 --- /dev/null +++ b/platformio/exception.py @@ -0,0 +1,110 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class PlatformioException(Exception): + MESSAGE = None + + def __str__(self): # pragma: no cover + if self.MESSAGE: + # pylint: disable=not-an-iterable + return self.MESSAGE.format(*self.args) + + return super().__str__() + + +class ReturnErrorCode(PlatformioException): + MESSAGE = "{0}" + + +class UserSideException(PlatformioException): + pass + + +class AbortedByUser(UserSideException): + MESSAGE = "Aborted by user" + + +# +# UDEV Rules +# + + +class InvalidUdevRules(UserSideException): + pass + + +class MissedUdevRules(InvalidUdevRules): + MESSAGE = ( + "Warning! Please install `99-platformio-udev.rules`. \nMore details: " + "https://docs.platformio.org/en/latest/core/installation/udev-rules.html" + ) + + +class OutdatedUdevRules(InvalidUdevRules): + MESSAGE = ( + "Warning! Your `{0}` are outdated. Please update or reinstall them." + "\nMore details: " + "https://docs.platformio.org/en/latest/core/installation/udev-rules.html" + ) + + +# +# Misc +# + + +class GetSerialPortsError(PlatformioException): + MESSAGE = "No implementation for your platform ('{0}') available" + + +class GetLatestVersionError(PlatformioException): + MESSAGE = "Can not retrieve the latest PlatformIO version" + + +class InvalidSettingName(UserSideException): + MESSAGE = "Invalid setting with the name '{0}'" + + +class InvalidSettingValue(UserSideException): + MESSAGE = "Invalid value '{0}' for the setting '{1}'" + + +class InvalidJSONFile(ValueError, UserSideException): + MESSAGE = "Could not load broken JSON: {0}" + + +class CIBuildEnvsEmpty(UserSideException): + MESSAGE = ( + "Can't find PlatformIO build environments.\n" + "Please specify `--board` or path to `platformio.ini` with " + "predefined environments using `--project-conf` option" + ) + + +class HomeDirPermissionsError(UserSideException): + MESSAGE = ( + "The directory `{0}` or its parent directory is not owned by the " + "current user and PlatformIO can not store configuration data.\n" + "Please check the permissions and owner of that directory.\n" + "Otherwise, please remove manually `{0}` directory and PlatformIO " + "will create new from the current user." + ) + + +class CygwinEnvDetected(PlatformioException): + MESSAGE = ( + "PlatformIO does not work within Cygwin environment. " + "Use native Terminal instead." + ) diff --git a/platformio/fs.py b/platformio/fs.py new file mode 100644 index 0000000..1f43a7a --- /dev/null +++ b/platformio/fs.py @@ -0,0 +1,226 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import hashlib +import io +import json +import os +import re +import shutil +import stat +import sys + +import click + +from platformio import exception +from platformio.compat import IS_WINDOWS + + +class cd: + def __init__(self, new_path): + self.new_path = new_path + self.prev_path = os.getcwd() + + def __enter__(self): + os.chdir(self.new_path) + + def __exit__(self, etype, value, traceback): + os.chdir(self.prev_path) + + +def get_source_dir(): + curpath = os.path.abspath(__file__) + if not os.path.isfile(curpath): + for p in sys.path: + if os.path.isfile(os.path.join(p, __file__)): + curpath = os.path.join(p, __file__) + break + return os.path.dirname(curpath) + + +def get_assets_dir(): + return os.path.join(get_source_dir(), "assets") + + +def load_json(file_path): + try: + with open(file_path, mode="r", encoding="utf8") as f: + return json.load(f) + except ValueError as exc: + raise exception.InvalidJSONFile(file_path) from exc + + +def humanize_file_size(filesize): + base = 1024 + unit = 0 + suffix = "B" + filesize = float(filesize) + if filesize < base: + return "%d%s" % (filesize, suffix) + for i, suffix in enumerate("KMGTPEZY"): + unit = base ** (i + 2) + if filesize >= unit: + continue + if filesize % (base ** (i + 1)): + return "%.2f%sB" % ((base * filesize / unit), suffix) + break + return "%d%sB" % ((base * filesize / unit), suffix) + + +def calculate_file_hashsum(algorithm, path): + h = hashlib.new(algorithm) + with io.open(path, "rb", buffering=0) as fp: + while True: + chunk = fp.read(io.DEFAULT_BUFFER_SIZE) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def calculate_folder_size(path): + assert os.path.isdir(path) + result = 0 + for root, __, files in os.walk(path): + for f in files: + file_path = os.path.join(root, f) + if not os.path.islink(file_path): + result += os.path.getsize(file_path) + return result + + +def get_platformio_udev_rules_path(): + return os.path.abspath( + os.path.join(get_assets_dir(), "system", "99-platformio-udev.rules") + ) + + +def ensure_udev_rules(): + from platformio.util import get_systype # pylint: disable=import-outside-toplevel + + def _rules_to_set(rules_path): + result = set() + with open(rules_path, encoding="utf8") as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + result.add(line) + return result + + if "linux" not in get_systype(): + return None + installed_rules = [ + "/etc/udev/rules.d/99-platformio-udev.rules", + "/lib/udev/rules.d/99-platformio-udev.rules", + ] + if not any(os.path.isfile(p) for p in installed_rules): + raise exception.MissedUdevRules + + origin_path = get_platformio_udev_rules_path() + if not os.path.isfile(origin_path): + return None + + origin_rules = _rules_to_set(origin_path) + for rules_path in installed_rules: + if not os.path.isfile(rules_path): + continue + current_rules = _rules_to_set(rules_path) + if not origin_rules <= current_rules: + raise exception.OutdatedUdevRules(rules_path) + + return True + + +def path_endswith_ext(path, extensions): + if not isinstance(extensions, (list, tuple)): + extensions = [extensions] + for ext in extensions: + if path.endswith("." + ext): + return True + return False + + +def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True): + def _add_candidate(items, item, src_dir): + if not src_exts or path_endswith_ext(item, src_exts): + items.add(os.path.relpath(item, src_dir)) + + def _find_candidates(pattern): + candidates = set() + for item in glob.glob( + os.path.join(glob.escape(src_dir), pattern), recursive=True + ): + if not os.path.isdir(item): + _add_candidate(candidates, item, src_dir) + continue + for root, dirs, files in os.walk(item, followlinks=followlinks): + for d in dirs if not followlinks else []: + if os.path.islink(os.path.join(root, d)): + _add_candidate(candidates, os.path.join(root, d), src_dir) + for f in files: + _add_candidate(candidates, os.path.join(root, f), src_dir) + return candidates + + src_filter = src_filter or "" + if isinstance(src_filter, (list, tuple)): + src_filter = " ".join(src_filter) + + result = set() + # correct fs directory separator + src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) + for action, pattern in re.findall(r"(\+|\-)<([^>]+)>", src_filter): + candidates = _find_candidates(pattern) + if action == "+": + result |= candidates + else: + result -= candidates + return sorted(list(result)) + + +def to_unix_path(path): + if not IS_WINDOWS or not path: + return path + return path.replace("\\", "/") + + +def expanduser(path): + """ + Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE + """ + if not IS_WINDOWS or not path.startswith("~") or "USERPROFILE" not in os.environ: + return os.path.expanduser(path) + return os.environ["USERPROFILE"] + path[1:] + + +def change_filemtime(path, mtime): + os.utime(path, (mtime, mtime)) + + +def rmtree(path): + def _onerror(func, path, __): + try: + st_mode = os.stat(path).st_mode + if st_mode & stat.S_IREAD: + os.chmod(path, st_mode | stat.S_IWRITE) + func(path) + except Exception as exc: # pylint: disable=broad-except + click.secho( + "%s \nPlease manually remove the file `%s`" % (str(exc), path), + fg="red", + err=True, + ) + + return shutil.rmtree(path, onerror=_onerror) diff --git a/platformio/home/__init__.py b/platformio/home/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/home/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/home/cli.py b/platformio/home/cli.py new file mode 100644 index 0000000..318c78f --- /dev/null +++ b/platformio/home/cli.py @@ -0,0 +1,118 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mimetypes +import socket + +import click + +from platformio.compat import IS_WINDOWS +from platformio.home.run import run_server +from platformio.package.manager.core import get_core_package_dir + + +@click.command("home", short_help="GUI to manage PlatformIO") +@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") +@click.option( + "--host", + default="127.0.0.1", + help=( + "HTTP host, default=127.0.0.1. You can open PIO Home for inbound " + "connections with --host=0.0.0.0" + ), +) +@click.option("--no-open", is_flag=True) +@click.option( + "--shutdown-timeout", + default=0, + type=int, + help=( + "Automatically shutdown server on timeout (in seconds) when no clients " + "are connected. Default is 0 which means never auto shutdown" + ), +) +@click.option( + "--session-id", + help=( + "A unique session identifier to keep PIO Home isolated from other instances " + "and protect from 3rd party access" + ), +) +def cli(port, host, no_open, shutdown_timeout, session_id): + # hook for `platformio-node-helpers` + if host == "__do_not_start__": + # download all dependent packages + get_core_package_dir("contrib-piohome") + return + + # Ensure PIO Home mimetypes are known + mimetypes.add_type("text/html", ".html") + mimetypes.add_type("text/css", ".css") + mimetypes.add_type("application/javascript", ".js") + + home_url = "http://%s:%d%s" % ( + host, + port, + ("/session/%s/" % session_id) if session_id else "/", + ) + click.echo( + "\n".join( + [ + "", + " ___I_", + " /\\-_--\\ PlatformIO Home", + "/ \\_-__\\", + "|[]| [] | %s" % home_url, + "|__|____|__%s" % ("_" * len(home_url)), + ] + ) + ) + click.echo("") + click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url) + + if is_port_used(host, port): + click.secho( + "PlatformIO Home server is already started in another process.", fg="yellow" + ) + if not no_open: + click.launch(home_url) + return + + run_server( + host=host, + port=port, + no_open=no_open, + shutdown_timeout=shutdown_timeout, + home_url=home_url, + ) + + +def is_port_used(host, port): + socket.setdefaulttimeout(1) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if IS_WINDOWS: + try: + s.bind((host, port)) + s.close() + return False + except (OSError, socket.error): + pass + else: + try: + s.connect((host, port)) + s.close() + except socket.error: + return False + + return True diff --git a/platformio/home/rpc/__init__.py b/platformio/home/rpc/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/home/rpc/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/home/rpc/handlers/__init__.py b/platformio/home/rpc/handlers/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/home/rpc/handlers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/home/rpc/handlers/account.py b/platformio/home/rpc/handlers/account.py new file mode 100644 index 0000000..f7e5dad --- /dev/null +++ b/platformio/home/rpc/handlers/account.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ajsonrpc.core import JSONRPC20DispatchException + +from platformio.account.client import AccountClient +from platformio.home.rpc.handlers.base import BaseRPCHandler + + +class AccountRPC(BaseRPCHandler): + @staticmethod + def call_client(method, *args, **kwargs): + try: + client = AccountClient() + return getattr(client, method)(*args, **kwargs) + except Exception as exc: # pylint: disable=bare-except + raise JSONRPC20DispatchException( + code=5000, message="PIO Account Call Error", data=str(exc) + ) from exc diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py new file mode 100644 index 0000000..4b6195e --- /dev/null +++ b/platformio/home/rpc/handlers/app.py @@ -0,0 +1,86 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from platformio import __version__, app, fs, util +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.project.helpers import is_platformio_project + + +class AppRPC(BaseRPCHandler): + IGNORE_STORAGE_KEYS = [ + "cid", + "coreVersion", + "coreSystype", + "coreCaller", + "coreSettings", + "homeDir", + "projectsDir", + ] + + @staticmethod + def load_state(): + with app.State( + app.resolve_state_path("core_dir", "homestate.json"), lock=True + ) as state: + storage = state.get("storage", {}) + + # base data + caller_id = app.get_session_var("caller_id") + storage["cid"] = app.get_cid() + storage["coreVersion"] = __version__ + storage["coreSystype"] = util.get_systype() + storage["coreCaller"] = str(caller_id).lower() if caller_id else None + storage["coreSettings"] = { + name: { + "description": data["description"], + "default_value": data["value"], + "value": app.get_setting(name), + } + for name, data in app.DEFAULT_SETTINGS.items() + } + + storage["homeDir"] = fs.expanduser("~") + storage["projectsDir"] = storage["coreSettings"]["projects_dir"]["value"] + + # skip non-existing recent projects + storage["recentProjects"] = list( + set( + str(Path(p).resolve()) + for p in storage.get("recentProjects", []) + if is_platformio_project(p) + ) + ) + + state["storage"] = storage + state.modified = False # skip saving extra fields + return state.as_dict() + + @staticmethod + def get_state(): + return AppRPC.load_state() + + @staticmethod + def save_state(state): + with app.State( + app.resolve_state_path("core_dir", "homestate.json"), lock=True + ) as s: + s.clear() + s.update(state) + storage = s.get("storage", {}) + for k in AppRPC.IGNORE_STORAGE_KEYS: + if k in storage: + del storage[k] + return True diff --git a/platformio/home/rpc/handlers/base.py b/platformio/home/rpc/handlers/base.py new file mode 100644 index 0000000..7b3f5d8 --- /dev/null +++ b/platformio/home/rpc/handlers/base.py @@ -0,0 +1,17 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseRPCHandler: + factory = None diff --git a/platformio/home/rpc/handlers/ide.py b/platformio/home/rpc/handlers/ide.py new file mode 100644 index 0000000..643ee0c --- /dev/null +++ b/platformio/home/rpc/handlers/ide.py @@ -0,0 +1,86 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from pathlib import Path + +from ajsonrpc.core import JSONRPC20DispatchException + +from platformio.compat import aio_get_running_loop +from platformio.home.rpc.handlers.base import BaseRPCHandler + + +class IDERPC(BaseRPCHandler): + COMMAND_TIMEOUT = 1.5 # in seconds + + def __init__(self): + self._ide_queue = [] + self._cmd_queue = {} + + async def listen_commands(self): + f = aio_get_running_loop().create_future() + self._ide_queue.append(f) + self._process_commands() + return await f + + async def send_command(self, command, params=None): + cmd_id = f"ide-{command}-{time.time()}" + self._cmd_queue[cmd_id] = { + "method": command, + "params": params, + "time": time.time(), + "future": aio_get_running_loop().create_future(), + } + self._process_commands() + # in case if IDE agent has not been started + aio_get_running_loop().call_later( + self.COMMAND_TIMEOUT + 0.1, self._process_commands + ) + return await self._cmd_queue[cmd_id]["future"] + + def on_command_result(self, cmd_id, value): + if cmd_id not in self._cmd_queue: + return False + if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs": + value = [str(Path(p).resolve()) for p in value] + self._cmd_queue[cmd_id]["future"].set_result(value) + del self._cmd_queue[cmd_id] + return True + + def _process_commands(self): + for cmd_id in list(self._cmd_queue): + cmd_data = self._cmd_queue[cmd_id] + if cmd_data["future"].done(): + del self._cmd_queue[cmd_id] + continue + + if ( + not self._ide_queue + and (time.time() - cmd_data["time"]) > self.COMMAND_TIMEOUT + ): + cmd_data["future"].set_exception( + JSONRPC20DispatchException( + code=4005, message="PIO Home IDE agent is not started" + ) + ) + continue + + while self._ide_queue: + self._ide_queue.pop().set_result( + { + "id": cmd_id, + "method": cmd_data["method"], + "params": cmd_data["params"], + } + ) diff --git a/platformio/home/rpc/handlers/misc.py b/platformio/home/rpc/handlers/misc.py new file mode 100644 index 0000000..ab2508a --- /dev/null +++ b/platformio/home/rpc/handlers/misc.py @@ -0,0 +1,50 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import time + +from platformio.cache import ContentCache +from platformio.compat import aio_create_task +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.home.rpc.handlers.os import OSRPC + + +class MiscRPC(BaseRPCHandler): + async def load_latest_tweets(self, data_url): + cache_key = ContentCache.key_from_args(data_url, "tweets") + cache_valid = "180d" + with ContentCache() as cc: + cache_data = cc.get(cache_key) + if cache_data: + cache_data = json.loads(cache_data) + # automatically update cache in background every 12 hours + if cache_data["time"] < (time.time() - (3600 * 12)): + aio_create_task( + self._preload_latest_tweets(data_url, cache_key, cache_valid) + ) + return cache_data["result"] + + return await self._preload_latest_tweets(data_url, cache_key, cache_valid) + + @staticmethod + async def _preload_latest_tweets(data_url, cache_key, cache_valid): + result = json.loads((await OSRPC.fetch_content(data_url))) + with ContentCache() as cc: + cc.set( + cache_key, + json.dumps({"time": int(time.time()), "result": result}), + cache_valid, + ) + return result diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py new file mode 100644 index 0000000..b8dcfc9 --- /dev/null +++ b/platformio/home/rpc/handlers/os.py @@ -0,0 +1,172 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import io +import os +import shutil +from functools import cmp_to_key + +import click + +from platformio import fs +from platformio.cache import ContentCache +from platformio.compat import aio_to_thread +from platformio.device.list.util import list_logical_devices +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.http import HTTPSession, ensure_internet_on + + +class HTTPAsyncSession(HTTPSession): + async def request( # pylint: disable=signature-differs,invalid-overridden-method + self, *args, **kwargs + ): + func = super().request + return await aio_to_thread(func, *args, **kwargs) + + +class OSRPC(BaseRPCHandler): + _http_session = None + + @classmethod + async def fetch_content(cls, url, data=None, headers=None, cache_valid=None): + if not headers: + headers = { + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) " + "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 " + "Safari/603.3.8" + ) + } + cache_key = ContentCache.key_from_args(url, data) if cache_valid else None + with ContentCache() as cc: + if cache_key: + result = cc.get(cache_key) + if result is not None: + return result + + # check internet before and resolve issue with 60 seconds timeout + ensure_internet_on(raise_exception=True) + + if not cls._http_session: + cls._http_session = HTTPAsyncSession() + + if data: + r = await cls._http_session.post(url, data=data, headers=headers) + else: + r = await cls._http_session.get(url, headers=headers) + + r.raise_for_status() + result = r.text + if cache_valid: + with ContentCache() as cc: + cc.set(cache_key, result, cache_valid) + return result + + async def request_content(self, uri, data=None, headers=None, cache_valid=None): + if uri.startswith("http"): + return await self.fetch_content(uri, data, headers, cache_valid) + local_path = uri[7:] if uri.startswith("file://") else uri + with io.open(local_path, encoding="utf-8") as fp: + return fp.read() + return None + + @staticmethod + def open_url(url): + return click.launch(url) + + @staticmethod + def reveal_file(path): + return click.launch(path, locate=True) + + @staticmethod + def open_file(path): + return click.launch(path) + + @staticmethod + def call_path_module_func(name, args, **kwargs): + return getattr(os.path, name)(*args, **kwargs) + + @staticmethod + def get_path_separator(): + return os.sep + + @staticmethod + def is_file(path): + return os.path.isfile(path) + + @staticmethod + def is_dir(path): + return os.path.isdir(path) + + @staticmethod + def make_dirs(path): + return os.makedirs(path) + + @staticmethod + def get_file_mtime(path): + return os.path.getmtime(path) + + @staticmethod + def rename(src, dst): + return os.rename(src, dst) + + @staticmethod + def copy(src, dst): + return shutil.copytree(src, dst, symlinks=True) + + @staticmethod + def glob(pathnames, root=None): + if not isinstance(pathnames, list): + pathnames = [pathnames] + result = set() + for pathname in pathnames: + result |= set( + glob.glob( + os.path.join(root, pathname) if root else pathname, recursive=True + ) + ) + return list(result) + + @staticmethod + def list_dir(path): + def _cmp(x, y): + if x[1] and not y[1]: + return -1 + if not x[1] and y[1]: + return 1 + if x[0].lower() > y[0].lower(): + return 1 + if x[0].lower() < y[0].lower(): + return -1 + return 0 + + items = [] + if path.startswith("~"): + path = fs.expanduser(path) + if not os.path.isdir(path): + return items + for item in os.listdir(path): + try: + item_is_dir = os.path.isdir(os.path.join(path, item)) + if item_is_dir: + os.listdir(os.path.join(path, item)) + items.append((item, item_is_dir)) + except OSError: + pass + return sorted(items, key=cmp_to_key(_cmp)) + + @staticmethod + def get_logical_devices(): + return list_logical_devices() diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py new file mode 100644 index 0000000..b09338e --- /dev/null +++ b/platformio/home/rpc/handlers/piocore.py @@ -0,0 +1,229 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import functools +import io +import json +import os +import sys +import threading + +import click +from ajsonrpc.core import JSONRPC20DispatchException + +from platformio import __main__, __version__, app, fs, proc, util +from platformio.compat import ( + IS_WINDOWS, + aio_create_task, + aio_get_running_loop, + aio_to_thread, + get_locale_encoding, + is_bytes, +) +from platformio.exception import PlatformioException +from platformio.home.rpc.handlers.base import BaseRPCHandler + + +class PIOCoreProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future, on_data_callback=None): + self.exit_future = exit_future + self.on_data_callback = on_data_callback + self.stdout = "" + self.stderr = "" + self._is_exited = False + self._encoding = get_locale_encoding() + + def pipe_data_received(self, fd, data): + data = data.decode(self._encoding, "replace") + pipe = ["stdin", "stdout", "stderr"][fd] + if pipe == "stdout": + self.stdout += data + if pipe == "stderr": + self.stderr += data + if self.on_data_callback: + self.on_data_callback(pipe=pipe, data=data) + + def connection_lost(self, exc): + self.process_exited() + + def process_exited(self): + if self._is_exited: + return + self.exit_future.set_result(True) + self._is_exited = True + + +class MultiThreadingStdStream: + def __init__(self, parent_stream): + self._buffers = {threading.get_ident(): parent_stream} + + def __getattr__(self, name): + thread_id = threading.get_ident() + self._ensure_thread_buffer(thread_id) + return getattr(self._buffers[thread_id], name) + + def _ensure_thread_buffer(self, thread_id): + if thread_id not in self._buffers: + self._buffers[thread_id] = io.StringIO() + + def write(self, value): + thread_id = threading.get_ident() + self._ensure_thread_buffer(thread_id) + return self._buffers[thread_id].write( + value.decode() if is_bytes(value) else value + ) + + def get_value_and_reset(self): + result = "" + try: + result = self.getvalue() + self.seek(0) + self.truncate(0) + except AttributeError: + pass + return result + + +@util.memoized(expire="60s") +def get_core_fullpath(): + return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) + + +class PIOCoreRPC(BaseRPCHandler): + @staticmethod + def version(): + return __version__ + + async def exec(self, args, options=None): + loop = aio_get_running_loop() + exit_future = loop.create_future() + data_callback = functools.partial( + self._on_exec_data_received, exec_options=options + ) + if args[0] != "--caller" and app.get_session_var("caller_id"): + args = ["--caller", app.get_session_var("caller_id")] + args + transport, protocol = await loop.subprocess_exec( + lambda: PIOCoreProtocol(exit_future, data_callback), + get_core_fullpath(), + *args, + stdin=None, + **options.get("spawn", {}), + ) + await exit_future + transport.close() + return { + "stdout": protocol.stdout, + "stderr": protocol.stderr, + "returncode": transport.get_returncode(), + } + + def _on_exec_data_received(self, exec_options, pipe, data): + notification_method = exec_options.get(f"{pipe}NotificationMethod") + if not notification_method: + return + aio_create_task( + self.factory.notify_clients( + method=notification_method, + params=[data], + actor="frontend", + ) + ) + + @staticmethod + def setup_multithreading_std_streams(): + if isinstance(sys.stdout, MultiThreadingStdStream): + return + PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout) + PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr) + sys.stdout = PIOCoreRPC.thread_stdout + sys.stderr = PIOCoreRPC.thread_stderr + + @staticmethod + async def call(args, options=None): + for i, arg in enumerate(args): + if not isinstance(arg, str): + args[i] = str(arg) + + options = options or {} + to_json = "--json-output" in args + + try: + if options.get("force_subprocess"): + result = await PIOCoreRPC._call_subprocess(args, options) + return PIOCoreRPC._process_result(result, to_json) + result = await PIOCoreRPC._call_inline(args, options) + try: + return PIOCoreRPC._process_result(result, to_json) + except ValueError: + # fall-back to subprocess method + result = await PIOCoreRPC._call_subprocess(args, options) + return PIOCoreRPC._process_result(result, to_json) + except Exception as exc: # pylint: disable=bare-except + raise JSONRPC20DispatchException( + code=5000, message="PIO Core Call Error", data=str(exc) + ) from exc + + @staticmethod + async def _call_subprocess(args, options): + result = await aio_to_thread( + proc.exec_command, + [get_core_fullpath()] + args, + cwd=options.get("cwd") or os.getcwd(), + ) + return (result["out"], result["err"], result["returncode"]) + + @staticmethod + async def _call_inline(args, options): + PIOCoreRPC.setup_multithreading_std_streams() + + def _thread_safe_call(args, cwd): + with fs.cd(cwd): + exit_code = __main__.main(["-c"] + args) + return ( + PIOCoreRPC.thread_stdout.get_value_and_reset(), + PIOCoreRPC.thread_stderr.get_value_and_reset(), + exit_code, + ) + + return await aio_to_thread( + _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd() + ) + + @staticmethod + def _process_result(result, to_json=False): + out, err, code = result + if out and is_bytes(out): + out = out.decode(get_locale_encoding()) + if err and is_bytes(err): + err = err.decode(get_locale_encoding()) + text = ("%s\n\n%s" % (out, err)).strip() + if code != 0: + raise PlatformioException(text) + if not to_json: + return text + try: + return json.loads(out) + except ValueError as exc: + click.secho("%s => `%s`" % (exc, out), fg="red", err=True) + # if PIO Core prints unhandled warnings + for line in out.split("\n"): + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except ValueError: + pass + raise exc diff --git a/platformio/home/rpc/handlers/platform.py b/platformio/home/rpc/handlers/platform.py new file mode 100644 index 0000000..34daf54 --- /dev/null +++ b/platformio/home/rpc/handlers/platform.py @@ -0,0 +1,140 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os.path + +from platformio.compat import aio_to_thread +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory + + +class PlatformRPC(BaseRPCHandler): + async def fetch_platforms(self, search_query=None, page=0, force_installed=False): + if force_installed: + return { + "items": await aio_to_thread( + self._load_installed_platforms, search_query + ) + } + + search_result = await self.factory.manager.dispatcher["registry.call_client"]( + method="list_packages", + query=search_query, + qualifiers={ + "types": ["platform"], + }, + page=page, + ) + return { + "page": search_result["page"], + "limit": search_result["limit"], + "total": search_result["total"], + "items": [ + { + "id": item["id"], + "ownername": item["owner"]["username"], + "name": item["name"], + "version": item["version"]["name"], + "description": item["description"], + "tier": item["tier"], + } + for item in search_result["items"] + ], + } + + @staticmethod + def _load_installed_platforms(search_query=None): + search_query = (search_query or "").strip() + + def _matchSearchQuery(p): + content_blocks = [p.name, p.title, p.description] + if p.frameworks: + content_blocks.append(" ".join(p.frameworks.keys())) + for board in p.get_boards().values(): + board_data = board.get_brief_data() + for key in ("id", "mcu", "vendor"): + content_blocks.append(board_data.get(key)) + return search_query in " ".join(content_blocks) + + items = [] + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) + if search_query and not _matchSearchQuery(p): + continue + items.append( + { + "__pkg_path": pkg.path, + "ownername": pkg.metadata.spec.owner if pkg.metadata.spec else None, + "name": p.name, + "version": str(pkg.metadata.version), + "title": p.title, + "description": p.description, + } + ) + return items + + async def fetch_boards(self, platform_spec): + spec = PackageSpec(platform_spec) + if spec.owner: + return await self.factory.manager.dispatcher["registry.call_client"]( + method="get_package", + typex="platform", + owner=spec.owner, + name=spec.name, + extra_path="/boards", + ) + return await aio_to_thread(self._load_installed_boards, spec) + + @staticmethod + def _load_installed_boards(platform_spec): + p = PlatformFactory.new(platform_spec) + return sorted( + [b.get_brief_data() for b in p.get_boards().values()], + key=lambda item: item["name"], + ) + + async def fetch_examples(self, platform_spec): + spec = PackageSpec(platform_spec) + if spec.owner: + return await self.factory.manager.dispatcher["registry.call_client"]( + method="get_package", + typex="platform", + owner=spec.owner, + name=spec.name, + extra_path="/examples", + ) + return await aio_to_thread(self._load_installed_examples, spec) + + @staticmethod + def _load_installed_examples(platform_spec): + platform = PlatformFactory.new(platform_spec) + platform_dir = platform.get_dir() + parser = ManifestParserFactory.new_from_dir(platform_dir) + result = parser.as_dict().get("examples") or [] + for example in result: + example["files"] = [ + { + "path": item, + "url": ( + "file://%s" + + os.path.join(platform_dir, "examples", example["name"], item) + ), + } + for item in example["files"] + ] + return result diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py new file mode 100644 index 0000000..2c03a7c --- /dev/null +++ b/platformio/home/rpc/handlers/project.py @@ -0,0 +1,386 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import time +from pathlib import Path + +import semantic_version +from ajsonrpc.core import JSONRPC20DispatchException + +from platformio import app, exception, fs +from platformio.home.rpc.handlers.app import AppRPC +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectError +from platformio.project.helpers import get_project_dir, is_platformio_project +from platformio.project.integration.generator import ProjectGenerator +from platformio.project.options import get_config_options_schema + + +class ProjectRPC(BaseRPCHandler): + @staticmethod + def config_call(init_kwargs, method, *args): + assert isinstance(init_kwargs, dict) + assert "path" in init_kwargs + if os.path.isdir(init_kwargs["path"]): + project_dir = init_kwargs["path"] + init_kwargs["path"] = os.path.join(init_kwargs["path"], "platformio.ini") + elif os.path.isfile(init_kwargs["path"]): + project_dir = os.path.dirname(init_kwargs["path"]) + else: + project_dir = get_project_dir() + with fs.cd(project_dir): + return getattr(ProjectConfig(**init_kwargs), method)(*args) + + @staticmethod + def config_load(path): + return ProjectConfig( + path, parse_extra=False, expand_interpolations=False + ).as_tuple() + + @staticmethod + def config_dump(path, data): + config = ProjectConfig(path, parse_extra=False, expand_interpolations=False) + config.update(data, clear=True) + return config.save() + + @staticmethod + def config_update_description(path, text): + config = ProjectConfig(path, parse_extra=False, expand_interpolations=False) + if not config.has_section("platformio"): + config.add_section("platformio") + if text: + config.set("platformio", "description", text) + else: + if config.has_option("platformio", "description"): + config.remove_option("platformio", "description") + if not config.options("platformio"): + config.remove_section("platformio") + return config.save() + + @staticmethod + def get_config_schema(): + return get_config_options_schema() + + @staticmethod + def get_projects(): + def _get_project_data(): + data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} + config = ProjectConfig() + data["envs"] = config.envs() + data["description"] = config.get("platformio", "description") + data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", [])) + + libdeps_dir = config.get("platformio", "libdeps_dir") + for section in config.sections(): + if not section.startswith("env:"): + continue + data["envLibdepsDirs"].append(os.path.join(libdeps_dir, section[4:])) + if config.has_option(section, "board"): + data["boards"].append(config.get(section, "board")) + data["libExtraDirs"].extend(config.get(section, "lib_extra_dirs", [])) + + # skip non existing folders and resolve full path + for key in ("envLibdepsDirs", "libExtraDirs"): + data[key] = [ + fs.expanduser(d) if d.startswith("~") else os.path.abspath(d) + for d in data[key] + if os.path.isdir(d) + ] + + return data + + def _path_to_name(path): + return (os.path.sep).join(path.split(os.path.sep)[-2:]) + + result = [] + pm = PlatformPackageManager() + for project_dir in AppRPC.load_state()["storage"]["recentProjects"]: + if not os.path.isdir(project_dir): + continue + data = {} + boards = [] + try: + with fs.cd(project_dir): + data = _get_project_data() + except ProjectError: + continue + + for board_id in data.get("boards", []): + name = board_id + try: + name = pm.board_config(board_id)["name"] + except exception.PlatformioException: + pass + boards.append({"id": board_id, "name": name}) + + result.append( + { + "path": project_dir, + "name": _path_to_name(project_dir), + "modified": int(os.path.getmtime(project_dir)), + "boards": boards, + "description": data.get("description"), + "envs": data.get("envs", []), + "envLibStorages": [ + {"name": os.path.basename(d), "path": d} + for d in data.get("envLibdepsDirs", []) + ], + "extraLibStorages": [ + {"name": _path_to_name(d), "path": d} + for d in data.get("libExtraDirs", []) + ], + } + ) + return result + + @staticmethod + def get_project_examples(): + result = [] + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + examples_dir = os.path.join(pkg.path, "examples") + if not os.path.isdir(examples_dir): + continue + items = [] + for project_dir, _, __ in os.walk(examples_dir): + project_description = None + try: + config = ProjectConfig(os.path.join(project_dir, "platformio.ini")) + config.validate(silent=True) + project_description = config.get("platformio", "description") + except ProjectError: + continue + + path_tokens = project_dir.split(os.path.sep) + items.append( + { + "name": "/".join( + path_tokens[path_tokens.index("examples") + 1 :] + ), + "path": project_dir, + "description": project_description, + } + ) + manifest = pm.load_manifest(pkg) + result.append( + { + "platform": { + "title": manifest["title"], + "version": manifest["version"], + }, + "items": sorted(items, key=lambda item: item["name"]), + } + ) + return sorted(result, key=lambda data: data["platform"]["title"]) + + async def init(self, board, framework, project_dir): + assert project_dir + if not os.path.isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--board", board, "--sample-code"] + if framework: + args.extend(["--project-option", "framework = %s" % framework]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + await PIOCoreRPC.call( + args, options={"cwd": project_dir, "force_subprocess": True} + ) + return project_dir + + @staticmethod + async def import_arduino(board, use_arduino_libs, arduino_project_dir): + board = str(board) + # don't import PIO Project + if is_platformio_project(arduino_project_dir): + return arduino_project_dir + + is_arduino_project = any( + os.path.isfile( + os.path.join( + arduino_project_dir, + "%s.%s" % (os.path.basename(arduino_project_dir), ext), + ) + ) + for ext in ("ino", "pde") + ) + if not is_arduino_project: + raise JSONRPC20DispatchException( + code=4000, message="Not an Arduino project: %s" % arduino_project_dir + ) + + state = AppRPC.load_state() + project_dir = os.path.join( + state["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + board + ) + if not os.path.isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--board", board] + args.extend(["--project-option", "framework = arduino"]) + if use_arduino_libs: + args.extend( + ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"] + ) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + await PIOCoreRPC.call( + args, options={"cwd": project_dir, "force_subprocess": True} + ) + with fs.cd(project_dir): + config = ProjectConfig() + src_dir = config.get("platformio", "src_dir") + if os.path.isdir(src_dir): + fs.rmtree(src_dir) + shutil.copytree(arduino_project_dir, src_dir, symlinks=True) + return project_dir + + @staticmethod + async def import_pio(project_dir): + if not project_dir or not is_platformio_project(project_dir): + raise JSONRPC20DispatchException( + code=4001, message="Not an PlatformIO project: %s" % project_dir + ) + new_project_dir = os.path.join( + AppRPC.load_state()["storage"]["projectsDir"], + time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir), + ) + shutil.copytree(project_dir, new_project_dir, symlinks=True) + + args = ["init"] + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + await PIOCoreRPC.call( + args, options={"cwd": new_project_dir, "force_subprocess": True} + ) + return new_project_dir + + async def init_v2(self, configuration, options=None): + project_dir = os.path.join(configuration["location"], configuration["name"]) + if not os.path.isdir(project_dir): + os.makedirs(project_dir) + + envclone = os.environ.copy() + envclone["PLATFORMIO_FORCE_ANSI"] = "true" + options = options or {} + options["spawn"] = {"env": envclone, "cwd": project_dir} + + args = ["project", "init"] + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + + if configuration.get("example"): + await self.factory.notify_clients( + method=options.get("stdoutNotificationMethod"), + params=["Copying example files...\n"], + actor="frontend", + ) + await self._pre_init_example(configuration, project_dir) + else: + args.extend(self._pre_init_empty(configuration)) + + return await self.factory.manager.dispatcher["core.exec"](args, options=options) + + @staticmethod + def _pre_init_empty(configuration): + project_options = [] + platform = configuration["platform"] + board_id = configuration.get("board", {}).get("id") + env_name = board_id or platform["name"] + if configuration.get("description"): + project_options.append(("description", configuration.get("description"))) + try: + v = semantic_version.Version(platform.get("version")) + assert not v.prerelease + project_options.append( + ("platform", "{name} @ ^{version}".format(**platform)) + ) + except (AssertionError, ValueError): + project_options.append( + ("platform", "{name} @ {version}".format(**platform)) + ) + if board_id: + project_options.append(("board", board_id)) + if configuration.get("framework"): + project_options.append(("framework", configuration["framework"]["name"])) + + args = ["-e", env_name, "--sample-code"] + for name, value in project_options: + args.extend(["-O", f"{name}={value}"]) + return args + + async def _pre_init_example(self, configuration, project_dir): + for item in configuration["example"]["files"]: + p = Path(project_dir).joinpath(item["path"]) + if not p.parent.is_dir(): + p.parent.mkdir(parents=True) + p.write_text( + await self.factory.manager.dispatcher["os.request_content"]( + item["url"] + ), + encoding="utf-8", + ) + return [] + + @staticmethod + def configuration(project_dir, env): + assert is_platformio_project(project_dir) + with fs.cd(project_dir): + config = ProjectConfig(os.path.join(project_dir, "platformio.ini")) + platform = PlatformFactory.from_env(env, autoinstall=True) + platform_pkg = PlatformPackageManager().get_package(platform.get_dir()) + board_id = config.get(f"env:{env}", "board", None) + + # frameworks + frameworks = [] + for name in config.get(f"env:{env}", "framework", []): + if name not in platform.frameworks: + continue + f_pkg_name = platform.frameworks[name].get("package") + if not f_pkg_name: + continue + f_pkg = platform.get_package(f_pkg_name) + if not f_pkg: + continue + f_manifest = platform.pm.load_manifest(f_pkg) + frameworks.append( + dict( + name=name, + title=f_manifest.get("title"), + version=str(f_pkg.metadata.version), + ) + ) + + return dict( + platform=dict( + ownername=platform_pkg.metadata.spec.owner + if platform_pkg.metadata.spec + else None, + name=platform.name, + title=platform.title, + version=str(platform_pkg.metadata.version), + ), + board=platform.board_config(board_id).get_brief_data() + if board_id + else None, + frameworks=frameworks or None, + ) diff --git a/platformio/home/rpc/handlers/registry.py b/platformio/home/rpc/handlers/registry.py new file mode 100644 index 0000000..84a374b --- /dev/null +++ b/platformio/home/rpc/handlers/registry.py @@ -0,0 +1,31 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ajsonrpc.core import JSONRPC20DispatchException + +from platformio.compat import aio_to_thread +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.registry.client import RegistryClient + + +class RegistryRPC(BaseRPCHandler): + @staticmethod + async def call_client(method, *args, **kwargs): + try: + client = RegistryClient() + return await aio_to_thread(getattr(client, method), *args, **kwargs) + except Exception as exc: # pylint: disable=bare-except + raise JSONRPC20DispatchException( + code=5000, message="Registry Call Error", data=str(exc) + ) from exc diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py new file mode 100644 index 0000000..ab7c13d --- /dev/null +++ b/platformio/home/rpc/server.py @@ -0,0 +1,131 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from urllib.parse import parse_qs + +import ajsonrpc.utils +import click +from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request +from ajsonrpc.dispatcher import Dispatcher +from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response +from starlette.endpoints import WebSocketEndpoint + +from platformio.compat import aio_create_task, aio_get_running_loop +from platformio.http import InternetConnectionError +from platformio.proc import force_exit + +# Remove this line when PR is merged +# https://github.com/pavlov99/ajsonrpc/pull/22 +ajsonrpc.utils.is_invalid_params = lambda: False + + +class JSONRPCServerFactoryBase: + connection_nums = 0 + shutdown_timer = None + + def __init__(self, shutdown_timeout=0): + self.shutdown_timeout = shutdown_timeout + self.manager = AsyncJSONRPCResponseManager( + Dispatcher(), is_server_error_verbose=True + ) + self._clients = {} + + def __call__(self, *args, **kwargs): + raise NotImplementedError + + def add_object_handler(self, handler, namespace): + handler.factory = self + self.manager.dispatcher.add_object(handler, prefix="%s." % namespace) + + def on_client_connect(self, connection, actor=None): + self._clients[connection] = {"actor": actor} + self.connection_nums += 1 + if self.shutdown_timer: + self.shutdown_timer.cancel() + self.shutdown_timer = None + + def on_client_disconnect(self, connection): + if connection in self._clients: + del self._clients[connection] + self.connection_nums -= 1 + if self.connection_nums < 1: + self.connection_nums = 0 + + if self.connection_nums == 0: + self.shutdown_by_timeout() + + async def on_shutdown(self): + pass + + def shutdown_by_timeout(self): + if self.shutdown_timeout < 1: + return + + def _auto_shutdown_server(): + click.echo("Automatically shutdown server on timeout") + force_exit() + + self.shutdown_timer = aio_get_running_loop().call_later( + self.shutdown_timeout, _auto_shutdown_server + ) + + async def notify_clients(self, method, params=None, actor=None): + for client, options in self._clients.items(): + if actor and options["actor"] != actor: + continue + request = JSONRPC20Request(method, params, is_notification=True) + await client.send_text(self.manager.serialize(request.body)) + return True + + +class WebSocketJSONRPCServerFactory(JSONRPCServerFactoryBase): + def __call__(self, *args, **kwargs): + ws = WebSocketJSONRPCServer(*args, **kwargs) + ws.factory = self + return ws + + +class WebSocketJSONRPCServer(WebSocketEndpoint): + encoding = "text" + factory: WebSocketJSONRPCServerFactory = None + + async def on_connect(self, websocket): + await websocket.accept() + qs = parse_qs(self.scope.get("query_string", b"")) + actors = qs.get(b"actor") + self.factory.on_client_connect( # pylint: disable=no-member + websocket, actor=actors[0].decode() if actors else None + ) + + async def on_receive(self, websocket, data): + aio_create_task(self._handle_rpc(websocket, data)) + + async def on_disconnect(self, websocket, close_code): + self.factory.on_client_disconnect(websocket) # pylint: disable=no-member + + async def _handle_rpc(self, websocket, data): + # pylint: disable=no-member + response = await self.factory.manager.get_response_for_payload(data) + if response.error and response.error.data: + click.secho("Error: %s" % response.error.data, fg="red", err=True) + if InternetConnectionError.MESSAGE in response.error.data: + response = JSONRPC20Response( + id=response.id, + error=JSONRPC20Error( + code=4008, + message="No Internet Connection", + data=response.error.data, + ), + ) + await websocket.send_text(self.factory.manager.serialize(response.body)) diff --git a/platformio/home/run.py b/platformio/home/run.py new file mode 100644 index 0000000..df3fa47 --- /dev/null +++ b/platformio/home/run.py @@ -0,0 +1,103 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from urllib.parse import urlparse + +import click +import uvicorn +from starlette.applications import Starlette +from starlette.middleware import Middleware +from starlette.responses import PlainTextResponse +from starlette.routing import Mount, Route, WebSocketRoute +from starlette.staticfiles import StaticFiles +from starlette.status import HTTP_403_FORBIDDEN + +from platformio.compat import aio_get_running_loop +from platformio.exception import PlatformioException +from platformio.home.rpc.handlers.account import AccountRPC +from platformio.home.rpc.handlers.app import AppRPC +from platformio.home.rpc.handlers.ide import IDERPC +from platformio.home.rpc.handlers.misc import MiscRPC +from platformio.home.rpc.handlers.os import OSRPC +from platformio.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.home.rpc.handlers.platform import PlatformRPC +from platformio.home.rpc.handlers.project import ProjectRPC +from platformio.home.rpc.handlers.registry import RegistryRPC +from platformio.home.rpc.server import WebSocketJSONRPCServerFactory +from platformio.package.manager.core import get_core_package_dir +from platformio.proc import force_exit + + +class ShutdownMiddleware: + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", ""): + await shutdown_server() + await self.app(scope, receive, send) + + +async def shutdown_server(_=None): + aio_get_running_loop().call_later(0.5, force_exit) + return PlainTextResponse("Server has been shutdown!") + + +async def protected_page(_): + return PlainTextResponse( + "Protected PlatformIO Home session", status_code=HTTP_403_FORBIDDEN + ) + + +def run_server(host, port, no_open, shutdown_timeout, home_url): + contrib_dir = get_core_package_dir("contrib-piohome") + if not os.path.isdir(contrib_dir): + raise PlatformioException("Invalid path to PIO Home Contrib") + + ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout) + ws_rpc_factory.add_object_handler(AccountRPC(), namespace="account") + ws_rpc_factory.add_object_handler(AppRPC(), namespace="app") + ws_rpc_factory.add_object_handler(IDERPC(), namespace="ide") + ws_rpc_factory.add_object_handler(MiscRPC(), namespace="misc") + ws_rpc_factory.add_object_handler(OSRPC(), namespace="os") + ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core") + ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project") + ws_rpc_factory.add_object_handler(PlatformRPC(), namespace="platform") + ws_rpc_factory.add_object_handler(RegistryRPC(), namespace="registry") + + path = urlparse(home_url).path + routes = [ + WebSocketRoute(path + "wsrpc", ws_rpc_factory, name="wsrpc"), + Route(path + "__shutdown__", shutdown_server, methods=["POST"]), + Mount(path, StaticFiles(directory=contrib_dir, html=True), name="static"), + ] + if path != "/": + routes.append(Route("/", protected_page)) + + uvicorn.run( + Starlette( + middleware=[Middleware(ShutdownMiddleware)], + routes=routes, + on_startup=[ + lambda: click.echo( + "PIO Home has been started. Press Ctrl+C to shutdown." + ), + lambda: None if no_open else click.launch(home_url), + ], + ), + host=host, + port=port, + log_level="warning", + ) diff --git a/platformio/http.py b/platformio/http.py new file mode 100644 index 0000000..ee67b63 --- /dev/null +++ b/platformio/http.py @@ -0,0 +1,221 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import socket +from urllib.parse import urljoin + +import requests.adapters +from urllib3.util.retry import Retry + +from platformio import __check_internet_hosts__, app, util +from platformio.cache import ContentCache, cleanup_content_cache +from platformio.exception import PlatformioException, UserSideException + +__default_requests_timeout__ = (10, None) # (connect, read) + + +class HTTPClientError(UserSideException): + def __init__(self, message, response=None): + super().__init__() + self.message = message + self.response = response + + def __str__(self): # pragma: no cover + return self.message + + +class InternetConnectionError(UserSideException): + MESSAGE = ( + "You are not connected to the Internet.\n" + "PlatformIO needs the Internet connection to" + " download dependent packages or to work with PlatformIO Account." + ) + + +class HTTPSession(requests.Session): + def __init__(self, *args, **kwargs): + self._x_base_url = kwargs.pop("x_base_url") if "x_base_url" in kwargs else None + super().__init__(*args, **kwargs) + self.headers.update({"User-Agent": app.get_user_agent()}) + try: + self.verify = app.get_setting("enable_proxy_strict_ssl") + except PlatformioException: + self.verify = True + + def request( # pylint: disable=signature-differs,arguments-differ + self, method, url, *args, **kwargs + ): + # print("HTTPSession::request", self._x_base_url, method, url, args, kwargs) + if "timeout" not in kwargs: + kwargs["timeout"] = __default_requests_timeout__ + return super().request( + method, + url + if url.startswith("http") or not self._x_base_url + else urljoin(self._x_base_url, url), + *args, + **kwargs + ) + + +class HTTPSessionIterator: + def __init__(self, endpoints): + if not isinstance(endpoints, list): + endpoints = [endpoints] + self.endpoints = endpoints + self.endpoints_iter = iter(endpoints) + # https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html + self.retry = Retry( + total=5, + backoff_factor=1, # [0, 2, 4, 8, 16] secs + # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], + status_forcelist=[413, 429, 500, 502, 503, 504], + ) + + def __iter__(self): # pylint: disable=non-iterator-returned + return self + + def __next__(self): + base_url = next(self.endpoints_iter) + session = HTTPSession(x_base_url=base_url) + adapter = requests.adapters.HTTPAdapter(max_retries=self.retry) + session.mount(base_url, adapter) + return session + + +class HTTPClient: + def __init__(self, endpoints): + self._session_iter = HTTPSessionIterator(endpoints) + self._session = None + self._next_session() + + def __del__(self): + if not self._session: + return + try: + self._session.close() + except: # pylint: disable=bare-except + pass + self._session = None + + def _next_session(self): + if self._session: + self._session.close() + self._session = next(self._session_iter) + + @util.throttle(500) + def send_request(self, method, path, **kwargs): + # check Internet before and resolve issue with 60 seconds timeout + ensure_internet_on(raise_exception=True) + + headers = kwargs.get("headers", {}) + with_authorization = ( + kwargs.pop("x_with_authorization") + if "x_with_authorization" in kwargs + else False + ) + if with_authorization and "Authorization" not in headers: + # pylint: disable=import-outside-toplevel + from platformio.account.client import AccountClient + + headers["Authorization"] = ( + "Bearer %s" % AccountClient().fetch_authentication_token() + ) + kwargs["headers"] = headers + + while True: + try: + return getattr(self._session, method)(path, **kwargs) + except requests.exceptions.RequestException as exc: + try: + self._next_session() + except Exception as exc2: + raise HTTPClientError(str(exc2)) from exc + + def fetch_json_data(self, method, path, **kwargs): + if method not in ("get", "head", "options"): + cleanup_content_cache("http") + cache_valid = kwargs.pop("x_cache_valid") if "x_cache_valid" in kwargs else None + if not cache_valid: + return self._parse_json_response(self.send_request(method, path, **kwargs)) + cache_key = ContentCache.key_from_args( + method, path, kwargs.get("params"), kwargs.get("data") + ) + with ContentCache("http") as cc: + result = cc.get(cache_key) + if result is not None: + try: + return json.loads(result) + except json.JSONDecodeError: + pass + response = self.send_request(method, path, **kwargs) + data = self._parse_json_response(response) + cc.set(cache_key, response.text, cache_valid) + return data + + @staticmethod + def _parse_json_response(response, expected_codes=(200, 201, 202)): + if response.status_code in expected_codes: + try: + return response.json() + except ValueError: + pass + try: + message = response.json()["message"] + except (KeyError, ValueError): + message = response.text + raise HTTPClientError(message, response) + + +# +# Helpers +# + + +@util.memoized(expire="10s") +def _internet_on(): + timeout = 2 + socket.setdefaulttimeout(timeout) + for host in __check_internet_hosts__: + try: + for var in ("HTTP_PROXY", "HTTPS_PROXY"): + if not os.getenv(var) and not os.getenv(var.lower()): + continue + requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) + return True + # try to resolve `host` for both AF_INET and AF_INET6, and then try to connect + # to all possible addresses (IPv4 and IPv6) in turn until a connection succeeds: + s = socket.create_connection((host, 80)) + s.close() + return True + except: # pylint: disable=bare-except + pass + return False + + +def ensure_internet_on(raise_exception=False): + result = _internet_on() + if raise_exception and not result: + raise InternetConnectionError() + return result + + +def fetch_remote_content(*args, **kwargs): + with HTTPSession() as s: + r = s.get(*args, **kwargs) + r.raise_for_status() + r.close() + return r.text diff --git a/platformio/maintenance.py b/platformio/maintenance.py new file mode 100644 index 0000000..45b4e5c --- /dev/null +++ b/platformio/maintenance.py @@ -0,0 +1,282 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +from time import time + +import click +import semantic_version + +from platformio import __version__, app, exception, fs, telemetry +from platformio.cache import cleanup_content_cache +from platformio.cli import PlatformioCLI +from platformio.commands.upgrade import get_latest_version +from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on +from platformio.package.manager.core import update_core_packages +from platformio.package.version import pepver_to_semver +from platformio.system.prune import calculate_unnecessary_system_data + + +def on_cmd_start(ctx, caller): + app.set_session_var("command_ctx", ctx) + set_caller(caller) + telemetry.on_cmd_start(ctx) + if PlatformioCLI.in_silence(): + return + after_upgrade(ctx) + + +def on_cmd_end(): + if PlatformioCLI.in_silence(): + return + + try: + check_platformio_upgrade() + check_prune_system() + except ( + HTTPClientError, + InternetConnectionError, + exception.GetLatestVersionError, + ): + click.secho( + "Failed to check for PlatformIO upgrades. " + "Please check your Internet connection.", + fg="red", + ) + + +def on_platformio_exception(exc): + telemetry.log_exception(exc) + + +def on_platformio_exit(): + telemetry.on_exit() + + +def set_caller(caller=None): + caller = caller or os.getenv("PLATFORMIO_CALLER") + if not caller: + if os.getenv("CODESPACES"): + caller = "codespaces" + elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): + caller = "vscode" + elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): + caller = "gitpod" + if caller: + app.set_session_var("caller_id", caller) + + +class Upgrader: + def __init__(self, from_version, to_version): + self.from_version = from_version + self.to_version = to_version + self._upgraders = [ + (semantic_version.Version("6.1.8-a.1"), self._appstate_migration), + ] + + def run(self, ctx): + if self.from_version > self.to_version: + return True + + result = [True] + for version, callback in self._upgraders: + if self.from_version >= version or self.to_version < version: + continue + result.append(callback(ctx)) + + return all(result) + + @staticmethod + def _appstate_migration(_): + state_path = app.resolve_state_path("core_dir", "appstate.json") + if not os.path.isfile(state_path): + return True + app.delete_state_item("telemetry") + created_at = app.get_state_item("created_at", None) + if not created_at: + state_stat = os.stat(state_path) + app.set_state_item( + "created_at", + int( + state_stat.st_birthtime + if hasattr(state_stat, "st_birthtime") + else state_stat.st_ctime + ), + ) + return True + + +def after_upgrade(ctx): + terminal_width = shutil.get_terminal_size().columns + last_version_str = app.get_state_item("last_version", "0.0.0") + if last_version_str == __version__: + return None + + if last_version_str == "0.0.0": + app.set_state_item("last_version", __version__) + return print_welcome_banner() + + last_version = pepver_to_semver(last_version_str) + current_version = pepver_to_semver(__version__) + + if last_version > current_version and not last_version.prerelease: + click.secho("*" * terminal_width, fg="yellow") + click.secho( + "Obsolete PIO Core v%s is used (previous was %s)" + % (__version__, last_version_str), + fg="yellow", + ) + click.secho("Please remove multiple PIO Cores from a system:", fg="yellow") + click.secho( + "https://docs.platformio.org/en/latest/core" + "/installation/troubleshooting.html", + fg="cyan", + ) + click.secho("*" * terminal_width, fg="yellow") + return None + + click.secho("Please wait while upgrading PlatformIO...", fg="yellow") + + # Update PlatformIO's Core packages + cleanup_content_cache("http") + update_core_packages() + + u = Upgrader(last_version, current_version) + if u.run(ctx): + app.set_state_item("last_version", __version__) + click.secho( + "PlatformIO has been successfully upgraded to %s!\n" % __version__, + fg="green", + ) + telemetry.log_event( + "pio_upgrade_core", + { + "label": "%s > %s" % (last_version_str, __version__), + "from_version": last_version_str, + "to_version": __version__, + }, + ) + + return print_welcome_banner() + + +def print_welcome_banner(): + terminal_width = shutil.get_terminal_size().columns + click.echo("*" * terminal_width) + click.echo("If you like %s, please:" % (click.style("PlatformIO", fg="cyan"))) + click.echo( + "- %s it on GitHub > %s" + % ( + click.style("star", fg="cyan"), + click.style("https://github.com/platformio/platformio-core", fg="cyan"), + ) + ) + click.echo( + "- %s us on LinkedIn to stay up-to-date " + "on the latest project news > %s" + % ( + click.style("follow", fg="cyan"), + click.style("https://www.linkedin.com/company/platformio/", fg="cyan"), + ) + ) + if not os.getenv("PLATFORMIO_IDE"): + click.echo( + "- %s PlatformIO IDE for embedded development > %s" + % ( + click.style("try", fg="cyan"), + click.style("https://platformio.org/platformio-ide", fg="cyan"), + ) + ) + + click.echo("*" * terminal_width) + click.echo("") + + +def check_platformio_upgrade(): + interval = int(app.get_setting("check_platformio_interval")) * 3600 * 24 + check_state = app.get_state_item("last_check", {}) + last_checked_time = check_state.get("platformio_upgrade", 0) + if (time() - interval) < last_checked_time: + return + + check_state["platformio_upgrade"] = int(time()) + app.set_state_item("last_check", check_state) + if not last_checked_time: + return + + ensure_internet_on(raise_exception=True) + + # Update PlatformIO Core packages + update_core_packages() + + latest_version = get_latest_version() + if pepver_to_semver(latest_version) <= pepver_to_semver(__version__): + return + + terminal_width = shutil.get_terminal_size().columns + + click.echo("") + click.echo("*" * terminal_width) + click.secho( + "There is a new version %s of PlatformIO available.\n" + "Please upgrade it via `" % latest_version, + fg="yellow", + nl=False, + ) + if os.path.join("Cellar", "platformio") in fs.get_source_dir(): + click.secho("brew update && brew upgrade", fg="cyan", nl=False) + click.secho("` command.", fg="yellow") + else: + click.secho("platformio upgrade", fg="cyan", nl=False) + click.secho("` or `", fg="yellow", nl=False) + click.secho("python -m pip install -U platformio", fg="cyan", nl=False) + click.secho("` command.", fg="yellow") + click.secho("Changes: ", fg="yellow", nl=False) + click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan") + click.echo("*" * terminal_width) + click.echo("") + + +def check_prune_system(): + interval = 30 * 3600 * 24 # 1 time per month + check_state = app.get_state_item("last_check", {}) + last_checked_time = check_state.get("prune_system", 0) + if (time() - interval) < last_checked_time: + return + + check_state["prune_system"] = int(time()) + app.set_state_item("last_check", check_state) + + if not last_checked_time: + return + + threshold_mb = int(app.get_setting("check_prune_system_threshold") or 0) + if threshold_mb <= 0: + return + + unnecessary_size = calculate_unnecessary_system_data() + if (unnecessary_size / 1024) < threshold_mb: + return + + terminal_width = shutil.get_terminal_size().columns + click.echo() + click.echo("*" * terminal_width) + click.secho( + "We found %s of unnecessary PlatformIO system data (temporary files, " + "unnecessary packages, etc.).\nUse `pio system prune --dry-run` to list " + "them or `pio system prune` to save disk space." + % fs.humanize_file_size(unnecessary_size), + fg="yellow", + ) diff --git a/platformio/package/__init__.py b/platformio/package/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/package/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/cli.py b/platformio/package/cli.py new file mode 100644 index 0000000..b2df9b6 --- /dev/null +++ b/platformio/package/cli.py @@ -0,0 +1,48 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.package.commands.exec import package_exec_cmd +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.list import package_list_cmd +from platformio.package.commands.outdated import package_outdated_cmd +from platformio.package.commands.pack import package_pack_cmd +from platformio.package.commands.publish import package_publish_cmd +from platformio.package.commands.search import package_search_cmd +from platformio.package.commands.show import package_show_cmd +from platformio.package.commands.uninstall import package_uninstall_cmd +from platformio.package.commands.unpublish import package_unpublish_cmd +from platformio.package.commands.update import package_update_cmd + + +@click.group( + "pkg", + commands=[ + package_exec_cmd, + package_install_cmd, + package_list_cmd, + package_outdated_cmd, + package_pack_cmd, + package_publish_cmd, + package_search_cmd, + package_show_cmd, + package_uninstall_cmd, + package_unpublish_cmd, + package_update_cmd, + ], + short_help="Unified Package Manager", +) +def cli(): + pass diff --git a/platformio/package/commands/__init__.py b/platformio/package/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/package/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/commands/exec.py b/platformio/package/commands/exec.py new file mode 100644 index 0000000..41386d5 --- /dev/null +++ b/platformio/package/commands/exec.py @@ -0,0 +1,109 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +import click + +from platformio.compat import IS_MACOS, IS_WINDOWS +from platformio.exception import ReturnErrorCode, UserSideException +from platformio.package.manager.tool import ToolPackageManager +from platformio.proc import get_pythonexe_path, where_is_program + + +@click.command("exec", short_help="Run command from package tool") +@click.option("-p", "--package", metavar="SPECIFICATION") +@click.option("-c", "--call", metavar=" [args...]") +@click.argument("args", nargs=-1, type=click.UNPROCESSED) +@click.pass_obj +def package_exec_cmd(obj, package, call, args): + if not call and not args: + raise click.BadArgumentUsage("Please provide command name") + pkg = None + if package: + pm = ToolPackageManager() + pkg = pm.get_package(package) + if not pkg: + pkg = pm.install(package) + else: + executable = args[0] if args else call.split(" ")[0] + pkg = find_pkg_by_executable(executable) + if not pkg: + raise UserSideException( + "Could not find a package with '%s' executable file" % executable + ) + + click.echo( + "Using %s package" + % click.style("%s@%s" % (pkg.metadata.name, pkg.metadata.version), fg="cyan") + ) + + inject_pkg_to_environ(pkg) + os.environ["PIO_PYTHON_EXE"] = get_pythonexe_path() + + # inject current python interpreter on Windows + if args[0].endswith(".py"): + args = [os.environ["PIO_PYTHON_EXE"]] + list(args) + if not os.path.exists(args[1]): + args[1] = where_is_program(args[1]) + + result = None + try: + run_options = dict(shell=call is not None, env=os.environ) + force_click_stream = (obj or {}).get("force_click_stream") + if force_click_stream: + run_options.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = subprocess.run( # pylint: disable=subprocess-run-check + call or args, **run_options + ) + if force_click_stream: + click.echo(result.stdout.decode().strip(), err=result.returncode != 0) + except Exception as exc: + raise UserSideException(exc) from exc + + if result and result.returncode != 0: + raise ReturnErrorCode(result.returncode) + + +def find_pkg_by_executable(executable): + exes = [executable] + if IS_WINDOWS and not executable.endswith(".exe"): + exes.append(f"{executable}.exe") + for pkg in ToolPackageManager().get_installed(): + for exe in exes: + if os.path.exists(os.path.join(pkg.path, exe)) or os.path.exists( + os.path.join(pkg.path, "bin", exe) + ): + return pkg + return None + + +def inject_pkg_to_environ(pkg): + bin_dir = os.path.join(pkg.path, "bin") + lib_dir = os.path.join(pkg.path, "lib") + + paths = [bin_dir, pkg.path] if os.path.isdir(bin_dir) else [pkg.path] + if os.environ.get("PATH"): + paths.append(os.environ.get("PATH")) + os.environ["PATH"] = os.pathsep.join(paths) + + if IS_WINDOWS or not os.path.isdir(lib_dir) or "toolchain" in pkg.metadata.name: + return + + lib_path_key = "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH" + lib_paths = [lib_dir] + if os.environ.get(lib_path_key): + lib_paths.append(os.environ.get(lib_path_key)) + os.environ[lib_path_key] = os.pathsep.join(lib_paths) diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py new file mode 100644 index 0000000..d297664 --- /dev/null +++ b/platformio/package/commands/install.py @@ -0,0 +1,336 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from pathlib import Path + +import click + +from platformio import fs +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.core import get_core_package_dir +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageCompatibility, PackageSpec +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies +from platformio.test.result import TestSuite +from platformio.test.runners.factory import TestRunnerFactory + + +@click.command( + "install", short_help="Install the project dependencies or custom packages" +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option("-e", "--environment", "environments", multiple=True) +@click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) +@click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True) +@click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True) +@click.option( + "--no-save", + is_flag=True, + help="Prevent saving specified packages to `platformio.ini`", +) +@click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies") +@click.option("-g", "--global", is_flag=True, help="Install package globally") +@click.option( + "--storage-dir", + default=None, + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Custom Package Manager storage for global packages", +) +@click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists") +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +def package_install_cmd(**options): + if options.get("global") or options.get("storage_dir"): + install_global_dependencies(options) + else: + install_project_dependencies(options) + + +def install_global_dependencies(options): + pm = PlatformPackageManager(options.get("storage_dir")) + tm = ToolPackageManager(options.get("storage_dir")) + lm = LibraryPackageManager(options.get("storage_dir")) + for obj in (pm, tm, lm): + obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG) + for spec in options.get("platforms"): + pm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + for spec in options.get("tools"): + tm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + for spec in options.get("libraries", []): + lm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + + +def install_project_dependencies(options): + environments = options["environments"] + with fs.cd(options["project_dir"]): + config = ProjectConfig.get_instance() + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + if not options.get("silent"): + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) + already_up_to_date = not install_project_env_dependencies(env, options) + if not options.get("silent") and already_up_to_date: + click.secho("Already up-to-date.", fg="green") + + +def install_project_env_dependencies(project_env, options=None): + """Used in `pio run` -> Processor""" + options = options or {} + installed_conds = [] + # custom platforms + if options.get("platforms"): + installed_conds.append( + _install_project_env_custom_platforms(project_env, options) + ) + # custom tools + if options.get("tools"): + installed_conds.append(_install_project_env_custom_tools(project_env, options)) + # custom libraries + if options.get("libraries"): + installed_conds.append( + _install_project_env_custom_libraries(project_env, options) + ) + # declared dependencies + if not installed_conds: + installed_conds = [ + _install_project_env_platform(project_env, options), + _install_project_env_libraries(project_env, options), + ] + return any(installed_conds) + + +def _install_project_env_platform(project_env, options): + config = ProjectConfig.get_instance() + pm = PlatformPackageManager() + if options.get("silent"): + pm.set_log_level(logging.WARN) + spec = config.get(f"env:{project_env}", "platform") + if not spec: + return False + already_up_to_date = not options.get("force") + if not pm.get_package(spec): + already_up_to_date = False + PlatformPackageManager().install( + spec, + project_env=project_env, + project_targets=options.get("project_targets"), + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + # ensure SCons is installed + get_core_package_dir("tool-scons") + return not already_up_to_date + + +def _install_project_env_custom_platforms(project_env, options): + already_up_to_date = not options.get("force") + pm = PlatformPackageManager() + if not options.get("silent"): + pm.set_log_level(logging.DEBUG) + for spec in options.get("platforms"): + if not pm.get_package(spec): + already_up_to_date = False + pm.install( + spec, + project_env=project_env, + project_targets=options.get("project_targets"), + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + return not already_up_to_date + + +def _install_project_env_custom_tools(project_env, options): + already_up_to_date = not options.get("force") + tm = ToolPackageManager() + if not options.get("silent"): + tm.set_log_level(logging.DEBUG) + specs_to_save = [] + for tool in options.get("tools"): + spec = PackageSpec(tool) + if not tm.get_package(spec): + already_up_to_date = False + pkg = tm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + specs_to_save.append(pkg_to_save_spec(pkg, spec)) + if not options.get("no_save") and specs_to_save: + save_project_dependencies( + os.getcwd(), + specs_to_save, + scope="platform_packages", + action="add", + environments=[project_env], + ) + return not already_up_to_date + + +def _install_project_env_libraries(project_env, options): + _uninstall_project_unused_libdeps(project_env, options) + already_up_to_date = not options.get("force") + config = ProjectConfig.get_instance() + + compatibility_qualifiers = {} + if config.get(f"env:{project_env}", "platform", None): + try: + p = PlatformFactory.new(config.get(f"env:{project_env}", "platform")) + compatibility_qualifiers["platforms"] = [p.name] + except UnknownPlatform: + pass + if config.get(f"env:{project_env}", "framework"): + compatibility_qualifiers["frameworks"] = config.get( + f"env:{project_env}", "framework" + ) + + env_lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), project_env), + compatibility=PackageCompatibility(**compatibility_qualifiers) + if compatibility_qualifiers + else None, + ) + private_lm = LibraryPackageManager( + os.path.join(config.get("platformio", "lib_dir")) + ) + if options.get("silent"): + env_lm.set_log_level(logging.WARN) + private_lm.set_log_level(logging.WARN) + + lib_deps = config.get(f"env:{project_env}", "lib_deps") + if "__test" in options.get("project_targets", []): + test_runner = TestRunnerFactory.new( + TestSuite(project_env, options.get("piotest_running_name", "*")), config + ) + lib_deps.extend(test_runner.EXTRA_LIB_DEPS or []) + + for library in lib_deps: + spec = PackageSpec(library) + # skip built-in dependencies + if not spec.external and not spec.owner: + continue + if not env_lm.get_package(spec): + already_up_to_date = False + env_lm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + + # install dependencies from the private libraries + for pkg in private_lm.get_installed(): + _install_project_private_library_deps(pkg, private_lm, env_lm, options) + + return not already_up_to_date + + +def _uninstall_project_unused_libdeps(project_env, options): + config = ProjectConfig.get_instance() + lib_deps = set(config.get(f"env:{project_env}", "lib_deps")) + if not lib_deps: + return + storage_dir = Path(config.get("platformio", "libdeps_dir"), project_env) + integrity_dat = storage_dir / "integrity.dat" + if integrity_dat.is_file(): + prev_lib_deps = set( + integrity_dat.read_text(encoding="utf-8").strip().split("\n") + ) + if lib_deps == prev_lib_deps: + return + lm = LibraryPackageManager(str(storage_dir)) + if options.get("silent"): + lm.set_log_level(logging.WARN) + else: + click.secho("Removing unused dependencies...") + for spec in set(prev_lib_deps) - set(lib_deps): + try: + lm.uninstall(spec) + except UnknownPackageError: + pass + if not storage_dir.is_dir(): + storage_dir.mkdir(parents=True) + integrity_dat.write_text("\n".join(lib_deps), encoding="utf-8") + + +def _install_project_private_library_deps(private_pkg, private_lm, env_lm, options): + for dependency in private_lm.get_pkg_dependencies(private_pkg) or []: + spec = private_lm.dependency_to_spec(dependency) + # skip built-in dependencies + if not spec.external and not spec.owner: + continue + pkg = private_lm.get_package(spec) + if not pkg and not env_lm.get_package(spec): + pkg = env_lm.install( + spec, + skip_dependencies=True, + force=options.get("force"), + ) + if not pkg: + continue + _install_project_private_library_deps(pkg, private_lm, env_lm, options) + + +def _install_project_env_custom_libraries(project_env, options): + already_up_to_date = not options.get("force") + config = ProjectConfig.get_instance() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), project_env) + ) + if not options.get("silent"): + lm.set_log_level(logging.DEBUG) + specs_to_save = [] + for library in options.get("libraries") or []: + spec = PackageSpec(library) + if not lm.get_package(spec): + already_up_to_date = False + pkg = lm.install( + spec, + skip_dependencies=options.get("skip_dependencies"), + force=options.get("force"), + ) + specs_to_save.append(pkg_to_save_spec(pkg, spec)) + if not options.get("no_save") and specs_to_save: + save_project_dependencies( + os.getcwd(), + specs_to_save, + scope="lib_deps", + action="add", + environments=[project_env], + ) + return not already_up_to_date diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py new file mode 100644 index 0000000..dd19973 --- /dev/null +++ b/platformio/package/commands/list.py @@ -0,0 +1,229 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import List + +import click + +from platformio import fs +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageItem, PackageSpec +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig + + +@click.command("list", short_help="List installed packages") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option("-e", "--environment", "environments", multiple=True) +@click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) +@click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True) +@click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True) +@click.option("-g", "--global", is_flag=True, help="List globally installed packages") +@click.option( + "--storage-dir", + default=None, + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Custom Package Manager storage for global packages", +) +@click.option("--only-platforms", is_flag=True, help="List only platform packages") +@click.option("--only-tools", is_flag=True, help="List only tool packages") +@click.option("--only-libraries", is_flag=True, help="List only library packages") +@click.option("-v", "--verbose", is_flag=True) +def package_list_cmd(**options): + if options.get("global"): + list_global_packages(options) + else: + list_project_packages(options) + + +def humanize_package(pkg, spec=None, verbose=False): + if spec and not isinstance(spec, PackageSpec): + spec = PackageSpec(spec) + data = [ + click.style(pkg.metadata.name, fg="cyan"), + click.style(f"@ {str(pkg.metadata.version)}", bold=True), + ] + extra_data = ["required: %s" % (spec.humanize() if spec else "Any")] + if verbose: + extra_data.append(pkg.path) + data.append("(%s)" % ", ".join(extra_data)) + return " ".join(data) + + +def print_dependency_tree(pm, specs=None, filter_specs=None, level=0, verbose=False): + filtered_pkgs = [ + pm.get_package(spec) for spec in filter_specs or [] if pm.get_package(spec) + ] + candidates = {} + if specs: + for spec in specs: + pkg = pm.get_package(spec) + if not pkg: + continue + candidates[pkg.path] = (pkg, spec) + else: + candidates = {pkg.path: (pkg, pkg.metadata.spec) for pkg in pm.get_installed()} + if not candidates: + return + candidates = sorted(candidates.values(), key=lambda item: item[0].metadata.name) + + for index, (pkg, spec) in enumerate(candidates): + if filtered_pkgs and not _pkg_tree_contains(pm, pkg, filtered_pkgs): + continue + printed_pkgs = pm.memcache_get("__printed_pkgs", []) + if printed_pkgs and pkg.path in printed_pkgs: + continue + printed_pkgs.append(pkg.path) + pm.memcache_set("__printed_pkgs", printed_pkgs) + + click.echo( + "%s%s %s" + % ( + "│ " * level, + "├──" if index < len(candidates) - 1 else "└──", + humanize_package( + pkg, + spec=spec, + verbose=verbose, + ), + ) + ) + + dependencies = pm.get_pkg_dependencies(pkg) + if dependencies: + print_dependency_tree( + pm, + specs=[pm.dependency_to_spec(item) for item in dependencies], + filter_specs=filter_specs, + level=level + 1, + verbose=verbose, + ) + + +def _pkg_tree_contains(pm, root: PackageItem, children: List[PackageItem]): + if root in children: + return True + for dependency in pm.get_pkg_dependencies(root) or []: + pkg = pm.get_package(pm.dependency_to_spec(dependency)) + if pkg and _pkg_tree_contains(pm, pkg, children): + return True + return False + + +def list_global_packages(options): + data = [ + ("platforms", PlatformPackageManager(options.get("storage_dir"))), + ("tools", ToolPackageManager(options.get("storage_dir"))), + ("libraries", LibraryPackageManager(options.get("storage_dir"))), + ] + only_packages = any( + options.get(typex) or options.get(f"only_{typex}") for (typex, _) in data + ) + for typex, pm in data: + skip_conds = [ + only_packages + and not options.get(typex) + and not options.get(f"only_{typex}"), + not pm.get_installed(), + ] + if any(skip_conds): + continue + click.secho(typex.capitalize(), bold=True) + print_dependency_tree( + pm, filter_specs=options.get(typex), verbose=options.get("verbose") + ) + click.echo() + + +def list_project_packages(options): + environments = options["environments"] + only_packages = any( + options.get(typex) or options.get(f"only_{typex}") + for typex in ("platforms", "tools", "libraries") + ) + only_platform_packages = any( + options.get(typex) or options.get(f"only_{typex}") + for typex in ("platforms", "tools") + ) + only_library_packages = options.get("libraries") or options.get("only_libraries") + + with fs.cd(options["project_dir"]): + config = ProjectConfig.get_instance() + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) + found = False + if not only_packages or only_platform_packages: + _found = print_project_env_platform_packages(env, options) + found = found or _found + if not only_packages or only_library_packages: + _found = print_project_env_library_packages(env, options) + found = found or _found + if not found: + click.echo("No packages") + if (not environments and len(config.envs()) > 1) or len(environments) > 1: + click.echo() + + +def print_project_env_platform_packages(project_env, options): + try: + p = PlatformFactory.from_env(project_env) + except UnknownPlatform: + return None + click.echo( + "Platform %s" + % ( + humanize_package( + PlatformPackageManager().get_package(p.get_dir()), + p.config.get(f"env:{project_env}", "platform"), + verbose=options.get("verbose"), + ) + ) + ) + print_dependency_tree( + p.pm, + specs=[p.get_package_spec(name) for name in p.packages], + filter_specs=options.get("tools"), + ) + click.echo() + return True + + +def print_project_env_library_packages(project_env, options): + config = ProjectConfig.get_instance() + lib_deps = config.get(f"env:{project_env}", "lib_deps") + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), project_env) + ) + if not lib_deps or not lm.get_installed(): + return None + click.echo("Libraries") + print_dependency_tree( + lm, + lib_deps, + filter_specs=options.get("libraries"), + verbose=options.get("verbose"), + ) + return True diff --git a/platformio/package/commands/outdated.py b/platformio/package/commands/outdated.py new file mode 100644 index 0000000..769c937 --- /dev/null +++ b/platformio/package/commands/outdated.py @@ -0,0 +1,216 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig + + +class OutdatedCandidate: + def __init__(self, pm, pkg, spec, envs=None): + self.pm = pm + self.pkg = pkg + self.spec = spec + self.envs = envs or [] + self.outdated = None + if not isinstance(self.envs, list): + self.envs = [self.envs] + + def __eq__(self, other): + return all( + [ + self.pm.package_dir == other.pm.package_dir, + self.pkg == other.pkg, + self.spec == other.spec, + ] + ) + + def check(self): + self.outdated = self.pm.outdated(self.pkg, self.spec) + + def is_outdated(self): + if not self.outdated: + self.check() + return self.outdated.is_outdated(allow_incompatible=self.pm.pkg_type != "tool") + + +@click.command("outdated", short_help="Check for outdated packages") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option("-e", "--environment", "environments", multiple=True) +def package_outdated_cmd(project_dir, environments): + with fs.cd(project_dir): + candidates = fetch_outdated_candidates(environments, with_progress=True) + print_outdated_candidates(candidates) + + +def print_outdated_candidates(candidates): + if not candidates: + click.secho("Everything is up-to-date!", fg="green") + return + tabulate_data = [ + ( + click.style( + candidate.pkg.metadata.name, + fg=get_candidate_update_color(candidate.outdated), + ), + candidate.outdated.current, + candidate.outdated.wanted, + click.style(candidate.outdated.latest, fg="cyan"), + candidate.pm.pkg_type.capitalize(), + ", ".join(set(candidate.envs)), + ) + for candidate in candidates + ] + click.echo() + click.secho("Semantic Versioning color legend:", bold=True) + click.echo( + tabulate( + [ + ( + click.style("", fg="red"), + "backward-incompatible updates", + ), + ( + click.style("", fg="yellow"), + "backward-compatible features", + ), + ( + click.style("", fg="green"), + "backward-compatible bug fixes", + ), + ], + tablefmt="plain", + ) + ) + click.echo() + click.echo( + tabulate( + tabulate_data, + headers=["Package", "Current", "Wanted", "Latest", "Type", "Environments"], + ) + ) + + +def get_candidate_update_color(outdated): + if outdated.update_increment_type == outdated.UPDATE_INCREMENT_MAJOR: + return "red" + if outdated.update_increment_type == outdated.UPDATE_INCREMENT_MINOR: + return "yellow" + if outdated.update_increment_type == outdated.UPDATE_INCREMENT_PATCH: + return "green" + return None + + +def fetch_outdated_candidates(environments, with_progress=False): + candidates = [] + config = ProjectConfig.get_instance() + config.validate(environments) + + def _add_candidate(data): + new_candidate = OutdatedCandidate( + data["pm"], data["pkg"], data["spec"], data["env"] + ) + for candidate in candidates: + if candidate == new_candidate: + candidate.envs.append(data["env"]) + return + candidates.append(new_candidate) + + # platforms + for item in find_platform_candidates(config, environments): + _add_candidate(item) + # platform package dependencies + for dep_item in find_platform_dependency_candidates(item["env"]): + _add_candidate(dep_item) + + # libraries + for item in find_library_candidates(config, environments): + _add_candidate(item) + + result = [] + if not with_progress: + for candidate in candidates: + if candidate.is_outdated(): + result.append(candidate) + return result + + with click.progressbar(candidates, label="Checking") as pb: + for candidate in pb: + if candidate.is_outdated(): + result.append(candidate) + return result + + +def find_platform_candidates(config, environments): + result = [] + pm = PlatformPackageManager() + for env in config.envs(): + platform = config.get(f"env:{env}", "platform", None) + if not platform or (environments and env not in environments): + continue + spec = PackageSpec(platform) + pkg = pm.get_package(spec) + if not pkg: + continue + result.append(dict(env=env, pm=pm, pkg=pkg, spec=spec)) + return result + + +def find_platform_dependency_candidates(env): + result = [] + p = PlatformFactory.from_env(env) + for pkg in p.get_installed_packages(): + result.append( + dict( + env=env, + pm=p.pm, + pkg=pkg, + spec=p.get_package_spec(pkg.metadata.name), + ) + ) + return sorted(result, key=lambda item: item["pkg"].metadata.name) + + +def find_library_candidates(config, environments): + result = [] + for env in config.envs(): + if environments and env not in environments: + continue + package_dir = os.path.join(config.get("platformio", "libdeps_dir") or "", env) + lib_deps = [ + item for item in config.get(f"env:{env}", "lib_deps", []) if "/" in item + ] + if not os.path.isdir(package_dir) or not lib_deps: + continue + pm = LibraryPackageManager(package_dir) + for lib in lib_deps: + spec = PackageSpec(lib) + pkg = pm.get_package(spec) + if not pkg: + continue + result.append(dict(env=env, pm=pm, pkg=pkg, spec=spec)) + return sorted(result, key=lambda item: item["pkg"].metadata.name) diff --git a/platformio/package/commands/pack.py b/platformio/package/commands/pack.py new file mode 100644 index 0000000..e9cd0f0 --- /dev/null +++ b/platformio/package/commands/pack.py @@ -0,0 +1,45 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click + +from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError +from platformio.package.pack import PackagePacker + + +@click.command("pack", short_help="Create a tarball from a package") +@click.argument( + "package", + default=os.getcwd, + metavar="", + type=click.Path(exists=True, file_okay=True, dir_okay=True), +) +@click.option( + "-o", "--output", help="A destination path (folder or a full path to file)" +) +def package_pack_cmd(package, output): + p = PackagePacker(package) + archive_path = p.pack(output) + # validate manifest + try: + ManifestSchema().load_manifest( + ManifestParserFactory.new_from_archive(archive_path).as_dict() + ) + except ManifestValidationError as exc: + os.remove(archive_path) + raise exc + click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py new file mode 100644 index 0000000..c8b4ea7 --- /dev/null +++ b/platformio/package/commands/publish.py @@ -0,0 +1,206 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tarfile +import tempfile +from datetime import datetime + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.account.client import AccountClient +from platformio.compat import isascii +from platformio.exception import UserSideException +from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.manifest.schema import ManifestSchema +from platformio.package.meta import PackageType +from platformio.package.pack import PackagePacker +from platformio.package.unpack import FileUnpacker, TARArchiver +from platformio.registry.client import RegistryClient + + +def validate_datetime(ctx, param, value): # pylint: disable=unused-argument + if not value: + return value + try: + datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError as exc: + raise click.BadParameter(exc) + return value + + +@click.command("publish", short_help="Publish a package to the registry") +@click.argument( + "package", + default=os.getcwd, + metavar="", + type=click.Path(exists=True, file_okay=True, dir_okay=True), +) +@click.option( + "--owner", + help="PIO Account username (can be organization username). " + "Default is set to a username of the authorized PIO Account", +) +@click.option( + "--type", + "typex", + type=click.Choice(list(PackageType.items().values())), + help="Custom package type", +) +@click.option( + "--released-at", + callback=validate_datetime, + help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52", +) +@click.option("--private", is_flag=True, help="Restricted access (not a public)") +@click.option( + "--notify/--no-notify", + default=True, + help="Notify by email when package is processed", +) +@click.option( + "--no-interactive", + is_flag=True, + help="Do not show interactive prompt", +) +@click.option( + "--non-interactive", + is_flag=True, + help="Do not show interactive prompt", + hidden=True, +) +def package_publish_cmd( # pylint: disable=too-many-arguments, too-many-locals + package, owner, typex, released_at, private, notify, no_interactive, non_interactive +): + click.secho("Preparing a package...", fg="cyan") + no_interactive = no_interactive or non_interactive + owner = owner or AccountClient().get_logged_username() + do_not_pack = ( + not os.path.isdir(package) + and isinstance(FileUnpacker.new_archiver(package), TARArchiver) + and PackageType.from_archive(package) + ) + archive_path = None + with tempfile.TemporaryDirectory() as tmp_dir: # pylint: disable=no-member + # publish .tar.gz instantly without repacking + if do_not_pack: + archive_path = package + else: + with fs.cd(tmp_dir): + p = PackagePacker(package) + archive_path = p.pack() + + typex = typex or PackageType.from_archive(archive_path) + manifest = ManifestSchema().load_manifest( + ManifestParserFactory.new_from_archive(archive_path).as_dict() + ) + name = manifest.get("name") + version = manifest.get("version") + data = [ + ("Type:", typex), + ("Owner:", owner), + ("Name:", name), + ("Version:", version), + ("Size:", fs.humanize_file_size(os.path.getsize(archive_path))), + ] + if manifest.get("system"): + data.insert(len(data) - 1, ("System:", ", ".join(manifest.get("system")))) + click.echo(tabulate(data, tablefmt="plain")) + + # check files containing non-ascii chars + check_archive_file_names(archive_path) + + # look for duplicates + check_package_duplicates(owner, typex, name, version, manifest.get("system")) + + if not no_interactive: + click.confirm( + "Are you sure you want to publish the %s %s to the registry?\n" + % ( + typex, + click.style( + "%s/%s@%s" % (owner, name, version), + fg="cyan", + ), + ), + abort=True, + ) + + click.secho( + "The package publishing may take some time depending " + "on your Internet connection and the package size.", + fg="yellow", + ) + click.echo("Publishing...") + response = RegistryClient().publish_package( + owner, typex, archive_path, released_at, private, notify + ) + if not do_not_pack: + os.remove(archive_path) + click.secho(response.get("message"), fg="green") + + +def check_archive_file_names(archive_path): + with tarfile.open(archive_path, mode="r:gz") as tf: + for name in tf.getnames(): + if not isascii(name) or not name.isprintable(): + click.secho( + f"Warning! The `{name}` file contains non-ASCII chars and can " + "lead to the unpacking issues on a user machine", + fg="yellow", + ) + + +def check_package_duplicates( + owner, type, name, version, system +): # pylint: disable=redefined-builtin + found = False + items = ( + RegistryClient() + .list_packages(qualifiers=dict(types=[type], names=[name])) + .get("items") + ) + if not items: + return True + # duplicated version by owner / system + found = False + for item in items: + if item["owner"]["username"] != owner or item["version"]["name"] != version: + continue + if not system: + found = True + break + published_systems = [] + for f in item["version"]["files"]: + published_systems.extend(f.get("system", [])) + found = set(system).issubset(set(published_systems)) + if found: + raise UserSideException( + "The package `%s/%s@%s` is already published in the registry" + % (owner, name, version) + ) + other_owners = [ + item["owner"]["username"] + for item in items + if item["owner"]["username"] != owner + ] + if other_owners: + click.secho( + "\nWarning! A package with the name `%s` is already published by the next " + "owners: %s\n" % (name, ", ".join(other_owners)), + fg="yellow", + ) + return True diff --git a/platformio/package/commands/search.py b/platformio/package/commands/search.py new file mode 100644 index 0000000..ac71ef4 --- /dev/null +++ b/platformio/package/commands/search.py @@ -0,0 +1,77 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import click + +from platformio import util +from platformio.registry.client import RegistryClient + + +@click.command("search", short_help="Search for packages") +@click.argument("query") +@click.option("-p", "--page", type=click.IntRange(min=1)) +@click.option( + "-s", + "--sort", + type=click.Choice(["relevance", "popularity", "trending", "added", "updated"]), +) +def package_search_cmd(query, page, sort): + client = RegistryClient() + result = client.list_packages(query, page=page, sort=sort) + if not result["total"]: + click.secho("Nothing has been found by your request", fg="yellow") + click.echo( + "Try a less-specific search or use truncation (or wildcard) operator *" + ) + return + print_search_result(result) + + +def print_search_result(result): + click.echo( + "Found %d packages (page %d of %d)" + % ( + result["total"], + result["page"], + math.ceil(result["total"] / result["limit"]), + ) + ) + for item in result["items"]: + click.echo() + print_search_item(item) + + +def print_search_item(item): + click.echo( + "%s/%s" + % ( + click.style(item["owner"]["username"], fg="cyan"), + click.style(item["name"], fg="cyan", bold=True), + ) + ) + click.echo( + "%s • %s • Published on %s" + % ( + item["type"].capitalize() + if item["tier"] == "community" + else click.style( + ("%s %s" % (item["tier"], item["type"])).title(), bold=True + ), + item["version"]["name"], + util.parse_datetime(item["version"]["released_at"]).strftime("%c"), + ) + ) + click.echo(item["description"]) diff --git a/platformio/package/commands/show.py b/platformio/package/commands/show.py new file mode 100644 index 0000000..5be6790 --- /dev/null +++ b/platformio/package/commands/show.py @@ -0,0 +1,154 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from urllib.parse import quote + +import click +from tabulate import tabulate + +from platformio import fs, util +from platformio.exception import UserSideException +from platformio.package.manager._registry import PackageManagerRegistryMixin +from platformio.package.meta import PackageSpec, PackageType +from platformio.registry.client import RegistryClient + + +@click.command("show", short_help="Show package information") +@click.argument("spec", metavar="[/][@]") +@click.option( + "-t", + "--type", + "pkg_type", + type=click.Choice(list(PackageType.items().values())), + help="Package type", +) +def package_show_cmd(spec, pkg_type): + spec = PackageSpec(spec) + data = fetch_package_data(spec, pkg_type) + if not data: + raise UserSideException( + "Could not find '%s' package in the PlatormIO Registry" % spec.humanize() + ) + + click.echo() + click.echo( + "%s/%s" + % ( + click.style(data["owner"]["username"], fg="cyan"), + click.style(data["name"], fg="cyan", bold=True), + ) + ) + click.echo( + "%s • %s • %s • Published on %s" + % ( + data["type"].capitalize(), + data["version"]["name"], + "Private" if data.get("private") else "Public", + util.parse_datetime(data["version"]["released_at"]).strftime("%c"), + ) + ) + + # Description + click.echo() + click.echo(data["description"]) + + # Extra info + click.echo() + fields = [ + ("homepage", "Homepage"), + ("repository_url", "Repository"), + ("license", "License"), + ("popularity_rank", "Popularity"), + ("stars_count", "Stars"), + ("examples_count", "Examples"), + ("version.unpacked_size", "Installed Size"), + ("dependents_count", "Used By"), + ("dependencies_count", "Dependencies"), + ("platforms", "Compatible Platforms"), + ("frameworks", "Compatible Frameworks"), + ("keywords", "Keywords"), + ] + type_plural = "libraries" if data["type"] == "library" else (data["type"] + "s") + extra = [ + ( + "Registry", + click.style( + "https://registry.platformio.org/%s/%s/%s" + % (type_plural, data["owner"]["username"], quote(data["name"])), + fg="blue", + ), + ) + ] + for key, title in fields: + if "." in key: + k1, k2 = key.split(".") + value = data.get(k1, {}).get(k2) + else: + value = data.get(key) + if not value: + continue + if isinstance(value, list): + value = ", ".join(value) + elif key.endswith("_size"): + value = fs.humanize_file_size(value) + extra.append((title, value)) + click.echo(tabulate(extra)) + + # Versions + click.echo("") + table = tabulate( + [ + ( + version["name"], + fs.humanize_file_size(max(f["size"] for f in version["files"])), + util.parse_datetime(version["released_at"]), + ) + for version in data["versions"] + ], + headers=["Version", "Size", "Published"], + ) + click.echo(table) + click.echo("") + + +def fetch_package_data(spec, pkg_type=None): + assert isinstance(spec, PackageSpec) + client = RegistryClient() + if pkg_type and spec.owner and spec.name: + return client.get_package( + pkg_type, spec.owner, spec.name, version=spec.requirements + ) + qualifiers = {} + if spec.id: + qualifiers["ids"] = str(spec.id) + if spec.name: + qualifiers["names"] = spec.name.lower() + if pkg_type: + qualifiers["types"] = pkg_type + if spec.owner: + qualifiers["owners"] = spec.owner.lower() + packages = client.list_packages(qualifiers=qualifiers)["items"] + if not packages: + return None + if len(packages) > 1: + PackageManagerRegistryMixin.print_multi_package_issue( + click.echo, packages, spec + ) + return None + return client.get_package( + packages[0]["type"], + packages[0]["owner"]["username"], + packages[0]["name"], + version=spec.requirements, + ) diff --git a/platformio/package/commands/uninstall.py b/platformio/package/commands/uninstall.py new file mode 100644 index 0000000..2808d49 --- /dev/null +++ b/platformio/package/commands/uninstall.py @@ -0,0 +1,237 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import click + +from platformio import fs +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.project.config import ProjectConfig +from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies + + +@click.command( + "uninstall", short_help="Uninstall the project dependencies or custom packages" +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option("-e", "--environment", "environments", multiple=True) +@click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) +@click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True) +@click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True) +@click.option( + "--no-save", + is_flag=True, + help="Prevent removing specified packages from `platformio.ini`", +) +@click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies") +@click.option("-g", "--global", is_flag=True, help="Uninstall global packages") +@click.option( + "--storage-dir", + default=None, + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Custom Package Manager storage for global packages", +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +def package_uninstall_cmd(**options): + if options.get("global"): + uninstall_global_dependencies(options) + else: + uninstall_project_dependencies(options) + + +def uninstall_global_dependencies(options): + pm = PlatformPackageManager(options.get("storage_dir")) + tm = ToolPackageManager(options.get("storage_dir")) + lm = LibraryPackageManager(options.get("storage_dir")) + for obj in (pm, tm, lm): + obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG) + for spec in options.get("platforms"): + pm.uninstall( + spec, + skip_dependencies=options.get("skip_dependencies"), + ) + for spec in options.get("tools"): + tm.uninstall( + spec, + skip_dependencies=options.get("skip_dependencies"), + ) + for spec in options.get("libraries", []): + lm.uninstall( + spec, + skip_dependencies=options.get("skip_dependencies"), + ) + + +def uninstall_project_dependencies(options): + environments = options["environments"] + with fs.cd(options["project_dir"]): + config = ProjectConfig.get_instance() + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + if not options["silent"]: + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) + already_up_to_date = not uninstall_project_env_dependencies(env, options) + if not options["silent"] and already_up_to_date: + click.secho("Already up-to-date.", fg="green") + + +def uninstall_project_env_dependencies(project_env, options=None): + options = options or {} + uninstalled_conds = [] + # custom platforms + if options.get("platforms"): + uninstalled_conds.append( + _uninstall_project_env_custom_platforms(project_env, options) + ) + # custom tools + if options.get("tools"): + uninstalled_conds.append( + _uninstall_project_env_custom_tools(project_env, options) + ) + # custom ibraries + if options.get("libraries"): + uninstalled_conds.append( + _uninstall_project_env_custom_libraries(project_env, options) + ) + # declared dependencies + if not uninstalled_conds: + uninstalled_conds = [ + _uninstall_project_env_platform(project_env, options), + _uninstall_project_env_libraries(project_env, options), + ] + return any(uninstalled_conds) + + +def _uninstall_project_env_platform(project_env, options): + config = ProjectConfig.get_instance() + pm = PlatformPackageManager() + if options.get("silent"): + pm.set_log_level(logging.WARN) + spec = config.get(f"env:{project_env}", "platform") + if not spec: + return None + already_up_to_date = True + if not pm.get_package(spec): + return None + PlatformPackageManager().uninstall( + spec, + project_env=project_env, + skip_dependencies=options.get("skip_dependencies"), + ) + return not already_up_to_date + + +def _uninstall_project_env_custom_platforms(project_env, options): + already_up_to_date = True + pm = PlatformPackageManager() + if not options.get("silent"): + pm.set_log_level(logging.DEBUG) + for spec in options.get("platforms"): + if pm.get_package(spec): + already_up_to_date = False + pm.uninstall( + spec, + project_env=project_env, + skip_dependencies=options.get("skip_dependencies"), + ) + return not already_up_to_date + + +def _uninstall_project_env_custom_tools(project_env, options): + already_up_to_date = True + tm = ToolPackageManager() + if not options.get("silent"): + tm.set_log_level(logging.DEBUG) + specs_to_save = [] + for tool in options.get("tools"): + spec = PackageSpec(tool) + if tm.get_package(spec): + already_up_to_date = False + pkg = tm.uninstall( + spec, + skip_dependencies=options.get("skip_dependencies"), + ) + specs_to_save.append(pkg_to_save_spec(pkg, spec)) + if not options.get("no_save") and specs_to_save: + save_project_dependencies( + os.getcwd(), + specs_to_save, + scope="platform_packages", + action="remove", + environments=[project_env], + ) + return not already_up_to_date + + +def _uninstall_project_env_libraries(project_env, options): + already_up_to_date = True + config = ProjectConfig.get_instance() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), project_env) + ) + if options.get("silent"): + lm.set_log_level(logging.WARN) + for library in config.get(f"env:{project_env}", "lib_deps"): + spec = PackageSpec(library) + # skip built-in dependencies + if not spec.external and not spec.owner: + continue + if lm.get_package(spec): + already_up_to_date = False + lm.uninstall( + spec, + skip_dependencies=options.get("skip_dependencies"), + ) + return not already_up_to_date + + +def _uninstall_project_env_custom_libraries(project_env, options): + already_up_to_date = True + config = ProjectConfig.get_instance() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), project_env) + ) + if not options.get("silent"): + lm.set_log_level(logging.DEBUG) + specs_to_save = [] + for library in options.get("libraries") or []: + spec = PackageSpec(library) + if lm.get_package(spec): + already_up_to_date = False + pkg = lm.uninstall( + spec, + skip_dependencies=options.get("skip_dependencies"), + ) + specs_to_save.append(pkg_to_save_spec(pkg, spec)) + if not options.get("no_save") and specs_to_save: + save_project_dependencies( + os.getcwd(), + specs_to_save, + scope="lib_deps", + action="remove", + environments=[project_env], + ) + return not already_up_to_date diff --git a/platformio/package/commands/unpublish.py b/platformio/package/commands/unpublish.py new file mode 100644 index 0000000..7d0e633 --- /dev/null +++ b/platformio/package/commands/unpublish.py @@ -0,0 +1,46 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.account.client import AccountClient +from platformio.package.meta import PackageSpec, PackageType +from platformio.registry.client import RegistryClient + + +@click.command("unpublish", short_help="Remove a pushed package from the registry") +@click.argument( + "package", required=True, metavar="[/][@]" +) +@click.option( + "--type", + type=click.Choice(list(PackageType.items().values())), + default="library", + help="Package type, default is set to `library`", +) +@click.option( + "--undo", + is_flag=True, + help="Undo a remove, putting a version back into the registry", +) +def package_unpublish_cmd(package, type, undo): # pylint: disable=redefined-builtin + spec = PackageSpec(package) + response = RegistryClient().unpublish_package( + owner=spec.owner or AccountClient().get_logged_username(), + type=type, + name=spec.name, + version=str(spec.requirements), + undo=undo, + ) + click.secho(response.get("message"), fg="green") diff --git a/platformio/package/commands/update.py b/platformio/package/commands/update.py new file mode 100644 index 0000000..8788f7b --- /dev/null +++ b/platformio/package/commands/update.py @@ -0,0 +1,250 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import click + +from platformio import fs +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.project.config import ProjectConfig +from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies + + +@click.command( + "update", short_help="Update the project dependencies or custom packages" +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option("-e", "--environment", "environments", multiple=True) +@click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) +@click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True) +@click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True) +@click.option( + "--no-save", + is_flag=True, + help="Prevent saving specified packages to `platformio.ini`", +) +@click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies") +@click.option("-g", "--global", is_flag=True, help="Update global packages") +@click.option( + "--storage-dir", + default=None, + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Custom Package Manager storage for global packages", +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") +def package_update_cmd(**options): + if options.get("global"): + update_global_dependencies(options) + else: + update_project_dependencies(options) + + +def update_global_dependencies(options): + pm = PlatformPackageManager(options.get("storage_dir")) + tm = ToolPackageManager(options.get("storage_dir")) + lm = LibraryPackageManager(options.get("storage_dir")) + for obj in (pm, tm, lm): + obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG) + for spec in options.get("platforms"): + pm.update( + from_spec=spec, + to_spec=spec, + skip_dependencies=options.get("skip_dependencies"), + ) + for spec in options.get("tools"): + tm.update( + from_spec=spec, + to_spec=spec, + skip_dependencies=options.get("skip_dependencies"), + ) + for spec in options.get("libraries", []): + lm.update( + from_spec=spec, + to_spec=spec, + skip_dependencies=options.get("skip_dependencies"), + ) + + +def update_project_dependencies(options): + environments = options["environments"] + with fs.cd(options["project_dir"]): + config = ProjectConfig.get_instance() + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + if not options["silent"]: + click.echo("Resolving %s dependencies..." % click.style(env, fg="cyan")) + already_up_to_date = not update_project_env_dependencies(env, options) + if not options["silent"] and already_up_to_date: + click.secho("Already up-to-date.", fg="green") + + +def update_project_env_dependencies(project_env, options=None): + options = options or {} + updated_conds = [] + # custom platforms + if options.get("platforms"): + updated_conds.append(_update_project_env_custom_platforms(project_env, options)) + # custom tools + if options.get("tools"): + updated_conds.append(_update_project_env_custom_tools(project_env, options)) + # custom ibraries + if options.get("libraries"): + updated_conds.append(_update_project_env_custom_libraries(project_env, options)) + # declared dependencies + if not updated_conds: + updated_conds = [ + _update_project_env_platform(project_env, options), + _update_project_env_libraries(project_env, options), + ] + return any(updated_conds) + + +def _update_project_env_platform(project_env, options): + config = ProjectConfig.get_instance() + pm = PlatformPackageManager() + if options.get("silent"): + pm.set_log_level(logging.WARN) + spec = config.get(f"env:{project_env}", "platform") + if not spec: + return None + cur_pkg = pm.get_package(spec) + if not cur_pkg: + return None + new_pkg = PlatformPackageManager().update( + cur_pkg, + to_spec=spec, + project_env=project_env, + skip_dependencies=options.get("skip_dependencies"), + ) + return cur_pkg != new_pkg + + +def _update_project_env_custom_platforms(project_env, options): + already_up_to_date = True + pm = PlatformPackageManager() + if not options.get("silent"): + pm.set_log_level(logging.DEBUG) + for spec in options.get("platforms"): + cur_pkg = pm.get_package(spec) + new_pkg = pm.update( + cur_pkg, + to_spec=spec, + project_env=project_env, + skip_dependencies=options.get("skip_dependencies"), + ) + if cur_pkg != new_pkg: + already_up_to_date = False + return not already_up_to_date + + +def _update_project_env_custom_tools(project_env, options): + already_up_to_date = True + tm = ToolPackageManager() + if not options.get("silent"): + tm.set_log_level(logging.DEBUG) + + specs_to_save = [] + for tool in options.get("tools"): + spec = PackageSpec(tool) + cur_pkg = tm.get_package(spec) + new_pkg = tm.update( + cur_pkg, + to_spec=spec, + skip_dependencies=options.get("skip_dependencies"), + ) + if cur_pkg != new_pkg: + already_up_to_date = False + specs_to_save.append(pkg_to_save_spec(new_pkg, spec)) + + if not options.get("no_save") and specs_to_save: + save_project_dependencies( + os.getcwd(), + specs_to_save, + scope="platform_packages", + action="add", + environments=[project_env], + ) + + return not already_up_to_date + + +def _update_project_env_libraries(project_env, options): + already_up_to_date = True + config = ProjectConfig.get_instance() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), project_env) + ) + if options.get("silent"): + lm.set_log_level(logging.WARN) + for library in config.get(f"env:{project_env}", "lib_deps"): + spec = PackageSpec(library) + # skip built-in dependencies + if not spec.external and not spec.owner: + continue + cur_pkg = lm.get_package(spec) + if cur_pkg: + new_pkg = lm.update( + cur_pkg, + to_spec=spec, + skip_dependencies=options.get("skip_dependencies"), + ) + if cur_pkg != new_pkg: + already_up_to_date = False + return not already_up_to_date + + +def _update_project_env_custom_libraries(project_env, options): + already_up_to_date = True + config = ProjectConfig.get_instance() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), project_env) + ) + if not options.get("silent"): + lm.set_log_level(logging.DEBUG) + + specs_to_save = [] + for library in options.get("libraries") or []: + spec = PackageSpec(library) + cur_pkg = lm.get_package(spec) + new_pkg = lm.update( + cur_pkg, + to_spec=spec, + skip_dependencies=options.get("skip_dependencies"), + ) + if cur_pkg != new_pkg: + already_up_to_date = False + specs_to_save.append(pkg_to_save_spec(new_pkg, spec)) + + if not options.get("no_save") and specs_to_save: + save_project_dependencies( + os.getcwd(), + specs_to_save, + scope="lib_deps", + action="add", + environments=[project_env], + ) + + return not already_up_to_date diff --git a/platformio/package/download.py b/platformio/package/download.py new file mode 100644 index 0000000..17cc1f3 --- /dev/null +++ b/platformio/package/download.py @@ -0,0 +1,165 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +from email.utils import parsedate +from os.path import getsize, join +from time import mktime + +import click + +from platformio import fs +from platformio.compat import is_terminal +from platformio.http import HTTPSession +from platformio.package.exception import PackageException + + +class FileDownloader: + def __init__(self, url, dest_dir=None): + self._http_session = HTTPSession() + self._http_response = None + # make connection + self._http_response = self._http_session.get( + url, + stream=True, + ) + if self._http_response.status_code != 200: + raise PackageException( + "Got the unrecognized status code '{0}' when downloaded {1}".format( + self._http_response.status_code, url + ) + ) + + disposition = self._http_response.headers.get("content-disposition") + if disposition and "filename=" in disposition: + self._fname = ( + disposition[disposition.index("filename=") + 9 :] + .replace('"', "") + .replace("'", "") + ) + else: + self._fname = [p for p in url.split("/") if p][-1] + self._fname = str(self._fname) + self._destination = self._fname + if dest_dir: + self.set_destination(join(dest_dir, self._fname)) + + def set_destination(self, destination): + self._destination = destination + + def get_filepath(self): + return self._destination + + def get_lmtime(self): + return self._http_response.headers.get("last-modified") + + def get_size(self): + if "content-length" not in self._http_response.headers: + return -1 + return int(self._http_response.headers["content-length"]) + + def start(self, with_progress=True, silent=False): + label = "Downloading" + file_size = self.get_size() + itercontent = self._http_response.iter_content( + chunk_size=io.DEFAULT_BUFFER_SIZE + ) + try: + with open(self._destination, "wb") as fp: + if file_size == -1 or not with_progress or silent: + if not silent: + click.echo(f"{label}...") + for chunk in itercontent: + fp.write(chunk) + + elif not is_terminal(): + click.echo(f"{label} 0%", nl=False) + print_percent_step = 10 + printed_percents = 0 + downloaded_size = 0 + for chunk in itercontent: + fp.write(chunk) + downloaded_size += len(chunk) + if (downloaded_size / file_size * 100) >= ( + printed_percents + print_percent_step + ): + printed_percents += print_percent_step + click.echo(f" {printed_percents}%", nl=False) + click.echo("") + + else: + with click.progressbar( + length=file_size, + iterable=itercontent, + label=label, + update_min_steps=min( + 256 * 1024, file_size / 100 + ), # every 256Kb or less + ) as pb: + for chunk in pb: + pb.update(len(chunk)) + fp.write(chunk) + finally: + self._http_response.close() + self._http_session.close() + + if self.get_lmtime(): + self._preserve_filemtime(self.get_lmtime()) + + return True + + def verify(self, checksum=None): + _dlsize = getsize(self._destination) + if self.get_size() != -1 and _dlsize != self.get_size(): + raise PackageException( + ( + "The size ({0:d} bytes) of downloaded file '{1}' " + "is not equal to remote size ({2:d} bytes)" + ).format(_dlsize, self._fname, self.get_size()) + ) + if not checksum: + return True + + checksum_len = len(checksum) + hash_algo = None + if checksum_len == 32: + hash_algo = "md5" + elif checksum_len == 40: + hash_algo = "sha1" + elif checksum_len == 64: + hash_algo = "sha256" + + if not hash_algo: + raise PackageException( + "Could not determine checksum algorithm by %s" % checksum + ) + + dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination) + if checksum.lower() != dl_checksum.lower(): + raise PackageException( + "The checksum '{0}' of the downloaded file '{1}' " + "does not match to the remote '{2}'".format( + dl_checksum, self._fname, checksum + ) + ) + return True + + def _preserve_filemtime(self, lmdate): + lmtime = mktime(parsedate(lmdate)) + fs.change_filemtime(self._destination, lmtime) + + def __del__(self): + self._http_session.close() + if self._http_response: + self._http_response.close() diff --git a/platformio/package/exception.py b/platformio/package/exception.py new file mode 100644 index 0000000..ba896fa --- /dev/null +++ b/platformio/package/exception.py @@ -0,0 +1,68 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import util +from platformio.exception import UserSideException + + +class PackageException(UserSideException): + pass + + +class ManifestException(PackageException): + pass + + +class UnknownManifestError(ManifestException): + pass + + +class ManifestParserError(ManifestException): + pass + + +class ManifestValidationError(ManifestException): + def __init__(self, messages, data, valid_data): + super().__init__() + self.messages = messages + self.data = data + self.valid_data = valid_data + + def __str__(self): + return ( + "Invalid manifest fields: %s. \nPlease check specification -> " + "https://docs.platformio.org/page/librarymanager/config.html" + % self.messages + ) + + +class MissingPackageManifestError(ManifestException): + MESSAGE = "Could not find one of '{0}' manifest files in the package" + + +class UnknownPackageError(PackageException): + MESSAGE = ( + "Could not find the package with '{0}' requirements for your system '%s'" + % util.get_systype() + ) + + +class NotGlobalLibDir(PackageException): + MESSAGE = ( + "The `{0}` is not a PlatformIO project.\n\n" + "To manage libraries in global storage `{1}`,\n" + "please use `platformio lib --global {2}` or specify custom storage " + "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" + "Check `platformio lib --help` for details." + ) diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py new file mode 100644 index 0000000..bc2a434 --- /dev/null +++ b/platformio/package/lockfile.py @@ -0,0 +1,117 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from time import sleep, time + +from platformio.exception import UserSideException + +LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour +LOCKFILE_DELAY = 0.2 + +LOCKFILE_INTERFACE_FCNTL = 1 +LOCKFILE_INTERFACE_MSVCRT = 2 + +try: + import fcntl + + LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_FCNTL +except ImportError: + try: + import msvcrt + + LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_MSVCRT + except ImportError: + LOCKFILE_CURRENT_INTERFACE = None + + +class LockFileExists(UserSideException): + pass + + +class LockFileTimeoutError(UserSideException): + pass + + +class LockFile: + def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY): + self.timeout = timeout + self.delay = delay + self._lock_path = os.path.abspath(path) + ".lock" + self._fp = None + + def _lock(self): + if not LOCKFILE_CURRENT_INTERFACE and os.path.exists(self._lock_path): + # remove stale lock + if time() - os.path.getmtime(self._lock_path) > 10: + try: + os.remove(self._lock_path) + except: # pylint: disable=bare-except + pass + else: + raise LockFileExists + + self._fp = open( # pylint: disable=consider-using-with + self._lock_path, mode="w", encoding="utf8" + ) + try: + if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: + fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT: + msvcrt.locking( # pylint: disable=used-before-assignment + self._fp.fileno(), msvcrt.LK_NBLCK, 1 + ) + except (BlockingIOError, IOError) as exc: + self._fp.close() + self._fp = None + raise LockFileExists from exc + return True + + def _unlock(self): + if not self._fp: + return + if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: + fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN) + elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT: + msvcrt.locking(self._fp.fileno(), msvcrt.LK_UNLCK, 1) + self._fp.close() + self._fp = None + + def acquire(self): + elapsed = 0 + while elapsed < self.timeout: + try: + return self._lock() + except LockFileExists: + sleep(self.delay) + elapsed += self.delay + + raise LockFileTimeoutError() + + def release(self): + self._unlock() + if os.path.exists(self._lock_path): + try: + os.remove(self._lock_path) + except: # pylint: disable=bare-except + pass + + def __enter__(self): + self.acquire() + + def __exit__(self, type_, value, traceback): + self.release() + + def __del__(self): + self.release() diff --git a/platformio/package/manager/__init__.py b/platformio/package/manager/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/package/manager/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py new file mode 100644 index 0000000..bf84498 --- /dev/null +++ b/platformio/package/manager/_download.py @@ -0,0 +1,100 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import logging +import os +import tempfile +import time + +import click + +from platformio import app, compat, util +from platformio.package.download import FileDownloader +from platformio.package.lockfile import LockFile + + +class PackageManagerDownloadMixin: + DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month + + def compute_download_path(self, *args): + request_hash = hashlib.new("sha1") + for arg in args: + request_hash.update(compat.hashlib_encode_data(arg)) + dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest()) + return dl_path + + def get_download_usagedb_path(self): + return os.path.join(self.get_download_dir(), "usage.db") + + def set_download_utime(self, path, utime=None): + with app.State(self.get_download_usagedb_path(), lock=True) as state: + state[os.path.basename(path)] = int(time.time() if not utime else utime) + + @util.memoized(DOWNLOAD_CACHE_EXPIRE) + def cleanup_expired_downloads(self, _=None): + with app.State(self.get_download_usagedb_path(), lock=True) as state: + # remove outdated + for fname in list(state.keys()): + if state[fname] > (time.time() - self.DOWNLOAD_CACHE_EXPIRE): + continue + del state[fname] + dl_path = os.path.join(self.get_download_dir(), fname) + if os.path.isfile(dl_path): + os.remove(dl_path) + + def download(self, url, checksum=None): + silent = not self.log.isEnabledFor(logging.INFO) + dl_path = self.compute_download_path(url, checksum or "") + if os.path.isfile(dl_path): + self.set_download_utime(dl_path) + return dl_path + + with_progress = not app.is_disabled_progressbar() + tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir()) + try: + with LockFile(dl_path): + try: + fd = FileDownloader(url) + fd.set_destination(tmp_path) + fd.start(with_progress=with_progress, silent=silent) + except IOError as exc: + raise_error = not silent + if with_progress: + try: + fd = FileDownloader(url) + fd.set_destination(tmp_path) + fd.start(with_progress=False, silent=silent) + except IOError: + raise_error = True + if raise_error: + self.log.error( + click.style( + "Error: Please read https://bit.ly/package-manager-ioerror", + fg="red", + ) + ) + raise exc + if checksum: + fd.verify(checksum) + os.close(tmp_fd) + os.rename(tmp_path, dl_path) + finally: + if os.path.isfile(tmp_path): + os.close(tmp_fd) + os.remove(tmp_path) + + assert os.path.isfile(dl_path) + self.set_download_utime(dl_path) + return dl_path diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py new file mode 100644 index 0000000..fc4d5dd --- /dev/null +++ b/platformio/package/manager/_install.py @@ -0,0 +1,293 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import shutil +import tempfile + +import click + +from platformio import app, compat, fs, util +from platformio.package.exception import PackageException, UnknownPackageError +from platformio.package.meta import PackageCompatibility, PackageItem +from platformio.package.unpack import FileUnpacker +from platformio.package.vcsclient import VCSClientFactory + + +class PackageManagerInstallMixin: + _INSTALL_HISTORY = None # avoid circle dependencies + + @staticmethod + def unpack(src, dst): + with_progress = not app.is_disabled_progressbar() + try: + with FileUnpacker(src) as fu: + return fu.unpack(dst, with_progress=with_progress) + except IOError as exc: + if not with_progress: + raise exc + with FileUnpacker(src) as fu: + return fu.unpack(dst, with_progress=False) + + def install(self, spec, skip_dependencies=False, force=False): + try: + self.lock() + pkg = self._install(spec, skip_dependencies=skip_dependencies, force=force) + self.memcache_reset() + self.cleanup_expired_downloads() + return pkg + finally: + self.unlock() + + def _install( + self, + spec, + skip_dependencies=False, + force=False, + compatibility: PackageCompatibility = None, + ): + spec = self.ensure_spec(spec) + + # avoid circle dependencies + if not self._INSTALL_HISTORY: + self._INSTALL_HISTORY = {} + if not force and spec in self._INSTALL_HISTORY: + return self._INSTALL_HISTORY[spec] + + # check if package is already installed + pkg = self.get_package(spec) + + # if a forced installation + if pkg and force: + self.uninstall(pkg) + pkg = None + + if pkg: + # avoid RecursionError for circular_dependencies + self._INSTALL_HISTORY[spec] = pkg + + self.log.debug( + click.style( + "{name}@{version} is already installed".format( + **pkg.metadata.as_dict() + ), + fg="yellow", + ) + ) + # ensure package dependencies are installed + if not skip_dependencies: + self.install_dependencies(pkg, print_header=False) + return pkg + + self.log.info("Installing %s" % click.style(spec.humanize(), fg="cyan")) + + if spec.external: + pkg = self.install_from_uri(spec.uri, spec) + else: + pkg = self.install_from_registry( + spec, + search_qualifiers=compatibility.to_search_qualifiers() + if compatibility + else None, + ) + + if not pkg or not pkg.metadata: + raise PackageException( + "Could not install package '%s' for '%s' system" + % (spec.humanize(), util.get_systype()) + ) + + self.call_pkg_script(pkg, "postinstall") + + self.log.info( + click.style( + "{name}@{version} has been installed!".format(**pkg.metadata.as_dict()), + fg="green", + ) + ) + + self.memcache_reset() + # avoid RecursionError for circular_dependencies + self._INSTALL_HISTORY[spec] = pkg + + if not skip_dependencies: + self.install_dependencies(pkg) + + return pkg + + def install_dependencies(self, pkg, print_header=True): + assert isinstance(pkg, PackageItem) + dependencies = self.get_pkg_dependencies(pkg) + if not dependencies: + return + if print_header: + self.log.info("Resolving dependencies...") + for dependency in dependencies: + try: + self.install_dependency(dependency) + except UnknownPackageError: + if dependency.get("owner"): + self.log.warning( + click.style( + "Warning! Could not install `%s` dependency " + "for the`%s` package" % (dependency, pkg.metadata.name), + fg="yellow", + ) + ) + + def install_dependency(self, dependency): + dependency_compatibility = PackageCompatibility.from_dependency(dependency) + if self.compatibility and not dependency_compatibility.is_compatible( + self.compatibility + ): + self.log.debug( + click.style( + "Skip incompatible `%s` dependency with `%s`" + % (dependency, self.compatibility), + fg="yellow", + ) + ) + return None + return self._install( + spec=self.dependency_to_spec(dependency), + compatibility=dependency_compatibility, + ) + + def install_from_uri(self, uri, spec, checksum=None): + spec = self.ensure_spec(spec) + + if spec.symlink: + return self.install_symlink(spec) + + tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir()) + vcs = None + try: + if uri.startswith("file://"): + _uri = uri[7:] + if os.path.isfile(_uri): + self.unpack(_uri, tmp_dir) + else: + fs.rmtree(tmp_dir) + shutil.copytree(_uri, tmp_dir, symlinks=True) + elif uri.startswith(("http://", "https://")): + dl_path = self.download(uri, checksum) + assert os.path.isfile(dl_path) + self.unpack(dl_path, tmp_dir) + else: + vcs = VCSClientFactory.new(tmp_dir, uri) + assert vcs.export() + + root_dir = self.find_pkg_root(tmp_dir, spec) + pkg_item = PackageItem( + root_dir, + self.build_metadata( + root_dir, spec, vcs.get_current_revision() if vcs else None + ), + ) + pkg_item.dump_meta() + return self._install_tmp_pkg(pkg_item) + finally: + if os.path.isdir(tmp_dir): + try: + fs.rmtree(tmp_dir) + except: # pylint: disable=bare-except + pass + + def _install_tmp_pkg(self, tmp_pkg): + assert isinstance(tmp_pkg, PackageItem) + # validate package version and declared requirements + if ( + tmp_pkg.metadata.spec.requirements + and tmp_pkg.metadata.version not in tmp_pkg.metadata.spec.requirements + ): + raise PackageException( + "Package version %s doesn't satisfy requirements %s based on %s" + % ( + tmp_pkg.metadata.version, + tmp_pkg.metadata.spec.requirements, + tmp_pkg.metadata, + ) + ) + dst_pkg = PackageItem( + os.path.join(self.package_dir, tmp_pkg.get_safe_dirname()) + ) + + # what to do with existing package? + action = "overwrite" + if tmp_pkg.metadata.spec.has_custom_name(): + action = "overwrite" + dst_pkg = PackageItem( + os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) + ) + elif dst_pkg.metadata: + if dst_pkg.metadata.spec.external: + if dst_pkg.metadata.spec.uri != tmp_pkg.metadata.spec.uri: + action = "detach-existing" + elif ( + dst_pkg.metadata.version != tmp_pkg.metadata.version + or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner + ): + action = ( + "detach-existing" + if tmp_pkg.metadata.version > dst_pkg.metadata.version + else "detach-new" + ) + + def _cleanup_dir(path): + if os.path.isdir(path): + fs.rmtree(path) + + if action == "detach-existing": + target_dirname = "%s@%s" % ( + tmp_pkg.get_safe_dirname(), + dst_pkg.metadata.version, + ) + if dst_pkg.metadata.spec.uri: + target_dirname = "%s@src-%s" % ( + tmp_pkg.get_safe_dirname(), + hashlib.md5( + compat.hashlib_encode_data(dst_pkg.metadata.spec.uri) + ).hexdigest(), + ) + # move existing into the new place + pkg_dir = os.path.join(self.package_dir, target_dirname) + _cleanup_dir(pkg_dir) + shutil.copytree(dst_pkg.path, pkg_dir, symlinks=True) + # move new source to the destination location + _cleanup_dir(dst_pkg.path) + shutil.copytree(tmp_pkg.path, dst_pkg.path, symlinks=True) + return PackageItem(dst_pkg.path) + + if action == "detach-new": + target_dirname = "%s@%s" % ( + tmp_pkg.get_safe_dirname(), + tmp_pkg.metadata.version, + ) + if tmp_pkg.metadata.spec.external: + target_dirname = "%s@src-%s" % ( + tmp_pkg.get_safe_dirname(), + hashlib.md5( + compat.hashlib_encode_data(tmp_pkg.metadata.spec.uri) + ).hexdigest(), + ) + pkg_dir = os.path.join(self.package_dir, target_dirname) + _cleanup_dir(pkg_dir) + shutil.copytree(tmp_pkg.path, pkg_dir, symlinks=True) + return PackageItem(pkg_dir) + + # otherwise, overwrite existing + _cleanup_dir(dst_pkg.path) + shutil.copytree(tmp_pkg.path, dst_pkg.path, symlinks=True) + return PackageItem(dst_pkg.path) diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py new file mode 100644 index 0000000..4b02a1d --- /dev/null +++ b/platformio/package/manager/_legacy.py @@ -0,0 +1,61 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs +from platformio.package.meta import PackageItem, PackageSpec + + +class PackageManagerLegacyMixin: + def build_legacy_spec(self, pkg_dir): + # find src manifest + src_manifest_name = ".piopkgmanager.json" + src_manifest_path = None + for name in os.listdir(pkg_dir): + if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): + continue + src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) + break + + if src_manifest_path: + src_manifest = fs.load_json(src_manifest_path) + return PackageSpec( + name=src_manifest.get("name"), + uri=src_manifest.get("url"), + requirements=src_manifest.get("requirements"), + ) + + # fall back to a package manifest + manifest = self.load_manifest(pkg_dir) + return PackageSpec(name=manifest.get("name")) + + def legacy_load_manifest(self, pkg): + if not isinstance(pkg, PackageItem): + assert os.path.isdir(pkg) + pkg = PackageItem(pkg) + manifest = self.load_manifest(pkg) + manifest["__pkg_dir"] = pkg.path + for key in ("name", "version"): + if not manifest.get(key): + manifest[key] = str(getattr(pkg.metadata, key)) + if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external: + manifest["__src_url"] = pkg.metadata.spec.uri + manifest["version"] = str(pkg.metadata.version) + if pkg.metadata and pkg.metadata.spec.owner: + manifest["ownername"] = pkg.metadata.spec.owner + return manifest + + def legacy_get_installed(self): + return [self.legacy_load_manifest(pkg) for pkg in self.get_installed()] diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py new file mode 100644 index 0000000..4c339be --- /dev/null +++ b/platformio/package/manager/_registry.py @@ -0,0 +1,197 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import click + +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSpec +from platformio.package.version import cast_version_to_semver +from platformio.registry.client import RegistryClient +from platformio.registry.mirror import RegistryFileMirrorIterator + + +class PackageManagerRegistryMixin: + def install_from_registry(self, spec, search_qualifiers=None): + package = version = None + if spec.owner and spec.name and not search_qualifiers: + package = self.fetch_registry_package(spec) + if not package: + raise UnknownPackageError(spec.humanize()) + version = self.pick_best_registry_version(package["versions"], spec) + elif spec.id or spec.name: + packages = self.search_registry_packages(spec, search_qualifiers) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1: + self.print_multi_package_issue(self.log.warning, packages, spec) + package, version = self.find_best_registry_version(packages, spec) + + if not package or not version: + raise UnknownPackageError(spec.humanize()) + + pkgfile = self.pick_compatible_pkg_file(version["files"]) if version else None + if not pkgfile: + raise UnknownPackageError(spec.humanize()) + + for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]): + try: + return self.install_from_uri( + url, + PackageSpec( + owner=package["owner"]["username"], + id=package["id"], + name=package["name"], + ), + checksum or pkgfile["checksum"]["sha256"], + ) + except Exception as exc: # pylint: disable=broad-except + self.log.warning( + click.style("Warning! Package Mirror: %s" % exc, fg="yellow") + ) + self.log.warning( + click.style("Looking for another mirror...", fg="yellow") + ) + + return None + + def get_registry_client_instance(self): + if not self._registry_client: + self._registry_client = RegistryClient() + return self._registry_client + + def search_registry_packages(self, spec, qualifiers=None): + assert isinstance(spec, PackageSpec) + qualifiers = qualifiers or {} + if spec.id: + qualifiers["ids"] = str(spec.id) + else: + qualifiers["types"] = self.pkg_type + qualifiers["names"] = spec.name.lower() + if spec.owner: + qualifiers["owners"] = spec.owner.lower() + return self.get_registry_client_instance().list_packages(qualifiers=qualifiers)[ + "items" + ] + + def fetch_registry_package(self, spec): + assert isinstance(spec, PackageSpec) + result = None + regclient = self.get_registry_client_instance() + if spec.owner and spec.name: + result = regclient.get_package(self.pkg_type, spec.owner, spec.name) + if not result and (spec.id or (spec.name and not spec.owner)): + packages = self.search_registry_packages(spec) + if packages: + result = regclient.get_package( + self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"] + ) + if not result: + raise UnknownPackageError(spec.humanize()) + return result + + def reveal_registry_package_id(self, spec): + spec = self.ensure_spec(spec) + if spec.id: + return spec.id + packages = self.search_registry_packages(spec) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1: + self.print_multi_package_issue(self.log.warning, packages, spec) + self.log.info("") + return packages[0]["id"] + + @staticmethod + def print_multi_package_issue(print_func, packages, spec): + print_func( + click.style( + "Warning! More than one package has been found by ", fg="yellow" + ) + + click.style(spec.humanize(), fg="cyan") + + click.style(" requirements:", fg="yellow") + ) + + for item in packages: + print_func( + " - {owner}/{name}@{version}".format( + owner=click.style(item["owner"]["username"], fg="cyan"), + name=item["name"], + version=item["version"]["name"], + ) + ) + print_func( + click.style( + "Please specify detailed REQUIREMENTS using package owner and version " + "(shown above) to avoid name conflicts", + fg="yellow", + ) + ) + + def find_best_registry_version(self, packages, spec): + for package in packages: + # find compatible version within the latest package versions + version = self.pick_best_registry_version([package["version"]], spec) + if version: + return (package, version) + + # if the custom version requirements, check ALL package versions + version = self.pick_best_registry_version( + self.fetch_registry_package( + PackageSpec( + id=package["id"], + owner=package["owner"]["username"], + name=package["name"], + ) + ).get("versions"), + spec, + ) + if version: + return (package, version) + time.sleep(1) + return (None, None) + + def get_compatible_registry_versions(self, versions, spec=None, custom_system=None): + assert not spec or isinstance(spec, PackageSpec) + result = [] + for version in versions: + semver = cast_version_to_semver(version["name"]) + if spec and spec.requirements and semver not in spec.requirements: + continue + if not any( + self.is_system_compatible(f.get("system"), custom_system=custom_system) + for f in version["files"] + ): + continue + result.append(version) + return result + + def pick_best_registry_version(self, versions, spec=None, custom_system=None): + best = None + for version in self.get_compatible_registry_versions( + versions, spec, custom_system + ): + semver = cast_version_to_semver(version["name"]) + if not best or (semver > cast_version_to_semver(best["name"])): + best = version + return best + + def pick_compatible_pkg_file(self, version_files, custom_system=None): + for item in version_files: + if self.is_system_compatible( + item.get("system"), custom_system=custom_system + ): + return item + return None diff --git a/platformio/package/manager/_symlink.py b/platformio/package/manager/_symlink.py new file mode 100644 index 0000000..33ff03f --- /dev/null +++ b/platformio/package/manager/_symlink.py @@ -0,0 +1,71 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from platformio import fs +from platformio.package.exception import PackageException +from platformio.package.meta import PackageItem, PackageSpec + + +class PackageManagerSymlinkMixin: + @staticmethod + def is_symlink(path): + return path and path.endswith(".pio-link") and os.path.isfile(path) + + @classmethod + def resolve_symlink(cls, path): + assert cls.is_symlink(path) + data = fs.load_json(path) + spec = PackageSpec(**data["spec"]) + assert spec.symlink + pkg_dir = spec.uri[10:] + if not os.path.isabs(pkg_dir): + pkg_dir = os.path.normpath(os.path.join(data["cwd"], pkg_dir)) + return (pkg_dir if os.path.isdir(pkg_dir) else None, spec) + + def get_symlinked_package(self, path): + pkg_dir, spec = self.resolve_symlink(path) + if not pkg_dir: + return None + pkg = PackageItem(os.path.realpath(pkg_dir)) + if not pkg.metadata: + pkg.metadata = self.build_metadata(pkg.path, spec) + return pkg + + def install_symlink(self, spec): + assert spec.symlink + pkg_dir = spec.uri[10:] + if not os.path.isdir(pkg_dir): + raise PackageException( + f"Can not create a symbolic link for `{pkg_dir}`, not a directory" + ) + link_path = os.path.join( + self.package_dir, + "%s.pio-link" % (spec.name or os.path.basename(os.path.abspath(pkg_dir))), + ) + with open(link_path, mode="w", encoding="utf-8") as fp: + json.dump(dict(cwd=os.getcwd(), spec=spec.as_dict()), fp) + return self.get_symlinked_package(link_path) + + def uninstall_symlink(self, spec): + assert spec.symlink + for name in os.listdir(self.package_dir): + path = os.path.join(self.package_dir, name) + if not self.is_symlink(path): + continue + pkg = self.get_symlinked_package(path) + if pkg.metadata.spec.uri == spec.uri: + os.remove(path) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py new file mode 100644 index 0000000..b374fd0 --- /dev/null +++ b/platformio/package/manager/_uninstall.py @@ -0,0 +1,97 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil + +import click + +from platformio import fs +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageItem, PackageSpec + + +class PackageManagerUninstallMixin: + def uninstall(self, spec, skip_dependencies=False): + try: + self.lock() + return self._uninstall(spec, skip_dependencies) + finally: + self.unlock() + + def _uninstall(self, spec, skip_dependencies=False): + pkg = self.get_package(spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(spec) + + uninstalled_pkgs = self.memcache_get("__uninstalled_pkgs", []) + if uninstalled_pkgs and pkg.path in uninstalled_pkgs: + return pkg + uninstalled_pkgs.append(pkg.path) + self.memcache_set("__uninstalled_pkgs", uninstalled_pkgs) + + self.log.info( + "Removing %s @ %s" + % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version) + ) + + self.call_pkg_script(pkg, "preuninstall") + + # firstly, remove dependencies + if not skip_dependencies: + self.uninstall_dependencies(pkg) + + if pkg.metadata.spec.symlink: + self.uninstall_symlink(pkg.metadata.spec) + elif os.path.islink(pkg.path): + os.unlink(pkg.path) + else: + fs.rmtree(pkg.path) + self.memcache_reset() + + # unfix detached-package with the same name + detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) + if ( + detached_pkg + and "@" in detached_pkg.path + and not os.path.isdir( + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) + ) + ): + shutil.move( + detached_pkg.path, + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), + ) + self.memcache_reset() + + self.log.info( + click.style( + "{name}@{version} has been removed!".format(**pkg.metadata.as_dict()), + fg="green", + ) + ) + + return pkg + + def uninstall_dependencies(self, pkg): + assert isinstance(pkg, PackageItem) + dependencies = self.get_pkg_dependencies(pkg) + if not dependencies: + return + self.log.info("Removing dependencies...") + for dependency in dependencies: + pkg = self.get_package(self.dependency_to_spec(dependency)) + if not pkg: + continue + self._uninstall(pkg) diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py new file mode 100644 index 0000000..353e9e4 --- /dev/null +++ b/platformio/package/manager/_update.py @@ -0,0 +1,137 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click + +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec +from platformio.package.vcsclient import VCSBaseException, VCSClientFactory + + +class PackageManagerUpdateMixin: + def outdated(self, pkg, spec=None): + assert isinstance(pkg, PackageItem) + assert pkg.metadata + + if spec and not isinstance(spec, PackageSpec): + spec = PackageSpec(spec) + + if not os.path.isdir(pkg.path): + return PackageOutdatedResult(current=pkg.metadata.version) + + # skip detached package to a specific version + detached_conditions = [ + "@" in pkg.path, + pkg.metadata.spec and not pkg.metadata.spec.external, + not spec, + ] + if all(detached_conditions): + return PackageOutdatedResult(current=pkg.metadata.version, detached=True) + + latest = None + wanted = None + if pkg.metadata.spec.external: + latest = self._fetch_vcs_latest_version(pkg) + else: + try: + reg_pkg = self.fetch_registry_package(pkg.metadata.spec) + latest = ( + self.pick_best_registry_version(reg_pkg["versions"]) or {} + ).get("name") + if spec: + wanted = ( + self.pick_best_registry_version(reg_pkg["versions"], spec) or {} + ).get("name") + if not wanted: # wrong library + latest = None + except UnknownPackageError: + pass + + return PackageOutdatedResult( + current=pkg.metadata.version, latest=latest, wanted=wanted + ) + + def _fetch_vcs_latest_version(self, pkg): + vcs = None + try: + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri, silent=True) + except VCSBaseException: + return None + if not vcs.can_be_updated: + return None + + vcs_revision = vcs.get_latest_revision() + if not vcs_revision: + return None + + return str( + self.build_metadata( + pkg.path, pkg.metadata.spec, vcs_revision=vcs_revision + ).version + ) + + def update( + self, + from_spec, + to_spec=None, + skip_dependencies=False, + ): + pkg = self.get_package(from_spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(from_spec) + + outdated = self.outdated(pkg, to_spec) + if not outdated.is_outdated(allow_incompatible=False): + self.log.debug( + click.style( + "{name}@{version} is already up-to-date".format( + **pkg.metadata.as_dict() + ), + fg="yellow", + ) + ) + return pkg + + self.log.info( + "Updating %s @ %s" + % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version) + ) + try: + self.lock() + return self._update(pkg, outdated, skip_dependencies) + finally: + self.unlock() + + def _update(self, pkg, outdated, skip_dependencies=False): + if pkg.metadata.spec.external: + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.uri) + assert vcs.update() + pkg.metadata.version = self._fetch_vcs_latest_version(pkg) + pkg.dump_meta() + return pkg + + # uninstall existing version + self.uninstall(pkg, skip_dependencies=True) + + return self.install( + PackageSpec( + id=pkg.metadata.spec.id, + owner=pkg.metadata.spec.owner, + name=pkg.metadata.spec.name, + requirements=outdated.wanted or outdated.latest, + ), + skip_dependencies=skip_dependencies, + ) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py new file mode 100644 index 0000000..f197ea1 --- /dev/null +++ b/platformio/package/manager/base.py @@ -0,0 +1,341 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import subprocess +from datetime import datetime + +import click +import semantic_version + +from platformio import fs, util +from platformio.cli import PlatformioCLI +from platformio.compat import ci_strings_are_equal +from platformio.package.exception import ManifestException, MissingPackageManifestError +from platformio.package.lockfile import LockFile +from platformio.package.manager._download import PackageManagerDownloadMixin +from platformio.package.manager._install import PackageManagerInstallMixin +from platformio.package.manager._legacy import PackageManagerLegacyMixin +from platformio.package.manager._registry import PackageManagerRegistryMixin +from platformio.package.manager._symlink import PackageManagerSymlinkMixin +from platformio.package.manager._uninstall import PackageManagerUninstallMixin +from platformio.package.manager._update import PackageManagerUpdateMixin +from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.meta import ( + PackageItem, + PackageMetadata, + PackageSpec, + PackageType, +) +from platformio.proc import get_pythonexe_path +from platformio.project.helpers import get_project_cache_dir + + +class ClickLoggingHandler(logging.Handler): + def emit(self, record): + click.echo(self.format(record)) + + +class BasePackageManager( # pylint: disable=too-many-public-methods,too-many-instance-attributes + PackageManagerDownloadMixin, + PackageManagerRegistryMixin, + PackageManagerSymlinkMixin, + PackageManagerInstallMixin, + PackageManagerUninstallMixin, + PackageManagerUpdateMixin, + PackageManagerLegacyMixin, +): + _MEMORY_CACHE = {} + + def __init__(self, pkg_type, package_dir, compatibility=None): + self.pkg_type = pkg_type + self.package_dir = package_dir + self.compatibility = compatibility + self.log = self._setup_logger() + + self._MEMORY_CACHE = {} + self._lockfile = None + self._download_dir = None + self._tmp_dir = None + self._registry_client = None + + def __repr__(self): + return ( + f"{self.__class__.__name__} " + ) + + def _setup_logger(self): + logger = logging.getLogger(str(self.__class__.__name__).replace("Package", " ")) + logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(name)s: %(message)s") + sh = ClickLoggingHandler() + sh.setFormatter(formatter) + logger.handlers.clear() + logger.addHandler(sh) + return logger + + def set_log_level(self, level): + self.log.setLevel(level) + + def lock(self): + if self._lockfile: + return + self.ensure_dir_exists(os.path.dirname(self.package_dir)) + self._lockfile = LockFile(self.package_dir) + self.ensure_dir_exists(self.package_dir) + self._lockfile.acquire() + + def unlock(self): + if hasattr(self, "_lockfile") and self._lockfile: + self._lockfile.release() + self._lockfile = None + + def __del__(self): + self.unlock() + + def memcache_get(self, key, default=None): + return self._MEMORY_CACHE.get(key, default) + + def memcache_set(self, key, value): + self._MEMORY_CACHE[key] = value + + def memcache_reset(self): + self._MEMORY_CACHE.clear() + + @staticmethod + def is_system_compatible(value, custom_system=None): + if not value or "*" in value: + return True + return util.items_in_list(value, custom_system or util.get_systype()) + + @staticmethod + def ensure_dir_exists(path): + if not os.path.isdir(path): + os.makedirs(path) + assert os.path.isdir(path) + return path + + @staticmethod + def ensure_spec(spec): + return spec if isinstance(spec, PackageSpec) else PackageSpec(spec) + + @property + def manifest_names(self): + raise NotImplementedError + + def get_download_dir(self): + if not self._download_dir: + self._download_dir = self.ensure_dir_exists( + os.path.join(get_project_cache_dir(), "downloads") + ) + return self._download_dir + + def get_tmp_dir(self): + if not self._tmp_dir: + self._tmp_dir = self.ensure_dir_exists( + os.path.join(get_project_cache_dir(), "tmp") + ) + return self._tmp_dir + + def find_pkg_root(self, path, spec): # pylint: disable=unused-argument + if self.manifest_exists(path): + return path + for root, _, _ in os.walk(path): + if self.manifest_exists(root): + return root + raise MissingPackageManifestError(", ".join(self.manifest_names)) + + def get_manifest_path(self, pkg_dir): + if not os.path.isdir(pkg_dir): + return None + for name in self.manifest_names: + manifest_path = os.path.join(pkg_dir, name) + if os.path.isfile(manifest_path): + return manifest_path + return None + + def manifest_exists(self, pkg_dir): + return self.get_manifest_path(pkg_dir) + + def load_manifest(self, src): + path = src.path if isinstance(src, PackageItem) else src + cache_key = "load_manifest-%s" % path + result = self.memcache_get(cache_key) + if result: + return result + candidates = ( + [os.path.join(path, name) for name in self.manifest_names] + if os.path.isdir(path) + else [path] + ) + for item in candidates: + if not os.path.isfile(item): + continue + try: + result = ManifestParserFactory.new_from_file(item).as_dict() + self.memcache_set(cache_key, result) + return result + except ManifestException as exc: + if not PlatformioCLI.in_silence(): + self.log.warning(click.style(str(exc), fg="yellow")) + raise MissingPackageManifestError(", ".join(self.manifest_names)) + + @staticmethod + def generate_rand_version(): + return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") + + def build_metadata(self, pkg_dir, spec, vcs_revision=None): + manifest = self.load_manifest(pkg_dir) + metadata = PackageMetadata( + type=self.pkg_type, + name=manifest.get("name"), + version=manifest.get("version"), + spec=spec, + ) + if not metadata.name or spec.has_custom_name(): + metadata.name = spec.name + if vcs_revision: + metadata.version = "%s+sha.%s" % ( + metadata.version if metadata.version else "0.0.0", + vcs_revision, + ) + if not metadata.version: + metadata.version = self.generate_rand_version() + return metadata + + def get_installed(self): # pylint: disable=too-many-branches + if not os.path.isdir(self.package_dir): + return [] + + cache_key = "get_installed" + if self.memcache_get(cache_key): + return self.memcache_get(cache_key) + + result = [] + for name in sorted(os.listdir(self.package_dir)): + if name.startswith("_tmp_installing"): # legacy tmp folder + continue + pkg = None + path = os.path.join(self.package_dir, name) + if os.path.isdir(path): + pkg = PackageItem(path) + elif self.is_symlink(path): + pkg = self.get_symlinked_package(path) + if not pkg: + continue + if not pkg.metadata: + try: + spec = self.build_legacy_spec(pkg.path) + pkg.metadata = self.build_metadata(pkg.path, spec) + except MissingPackageManifestError: + pass + if not pkg.metadata: + continue + if self.pkg_type == PackageType.TOOL: + try: + if not self.is_system_compatible( + self.load_manifest(pkg).get("system") + ): + continue + except MissingPackageManifestError: + pass + result.append(pkg) + + self.memcache_set(cache_key, result) + return result + + def get_package(self, spec): + if isinstance(spec, PackageItem): + return spec + spec = self.ensure_spec(spec) + best = None + for pkg in self.get_installed(): + if not self.test_pkg_spec(pkg, spec): + continue + assert isinstance(pkg.metadata.version, semantic_version.Version) + if spec.requirements and pkg.metadata.version not in spec.requirements: + continue + if not best or (pkg.metadata.version > best.metadata.version): + best = pkg + return best + + @staticmethod + def test_pkg_spec(pkg, spec): + # "id" mismatch + if spec.id and spec.id != pkg.metadata.spec.id: + return False + + # external "URL" mismatch + if spec.external: + # local folder mismatch + if os.path.abspath(spec.uri) == os.path.abspath(pkg.path) or ( + spec.uri.startswith("file://") + and os.path.abspath(pkg.path) == os.path.abspath(spec.uri[7:]) + ): + return True + if spec.uri != pkg.metadata.spec.uri: + return False + + # "owner" mismatch + elif spec.owner and not ci_strings_are_equal( + spec.owner, pkg.metadata.spec.owner + ): + return False + + # "name" mismatch + elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): + return False + + return True + + def get_pkg_dependencies(self, pkg): + return self.load_manifest(pkg).get("dependencies") + + @staticmethod + def dependency_to_spec(dependency): + return PackageSpec( + owner=dependency.get("owner"), + name=dependency.get("name"), + requirements=dependency.get("version"), + ) + + def call_pkg_script(self, pkg, event): + manifest = None + try: + manifest = self.load_manifest(pkg) + except MissingPackageManifestError: + pass + scripts = (manifest or {}).get("scripts") + if not scripts or not isinstance(scripts, dict): + return + cmd = scripts.get(event) + if not cmd: + return + shell = False + if not isinstance(cmd, list): + shell = True + cmd = [cmd] + os.environ["PIO_PYTHON_EXE"] = get_pythonexe_path() + with fs.cd(pkg.path): + if os.path.isfile(cmd[0]) and cmd[0].endswith(".py"): + cmd = [os.environ["PIO_PYTHON_EXE"]] + cmd + subprocess.run( + " ".join(cmd) if shell else cmd, + cwd=pkg.path, + shell=shell, + env=os.environ, + check=True, + ) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py new file mode 100644 index 0000000..ca5c5e0 --- /dev/null +++ b/platformio/package/manager/core.py @@ -0,0 +1,93 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import __core_packages__, exception +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec + + +def get_installed_core_packages(): + result = [] + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if pkg: + result.append(pkg) + return result + + +def get_core_package_dir(name, spec=None, auto_install=True): + if name not in __core_packages__: + raise exception.PlatformioException("Please upgrade PlatformIO Core") + pm = ToolPackageManager() + spec = spec or PackageSpec( + owner="platformio", name=name, requirements=__core_packages__[name] + ) + pkg = pm.get_package(spec) + if pkg: + return pkg.path + if not auto_install: + return None + assert pm.install(spec) + remove_unnecessary_core_packages() + return pm.get_package(spec).path + + +def update_core_packages(): + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + try: + pm.update(spec, spec) + except UnknownPackageError: + pass + remove_unnecessary_core_packages() + return True + + +def remove_unnecessary_core_packages(dry_run=False): + candidates = [] + pm = ToolPackageManager() + best_pkg_versions = {} + + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: + continue + # pylint: disable=no-member + best_pkg_versions[pkg.metadata.name] = pkg.metadata.version + + for pkg in pm.get_installed(): + skip_conds = [ + os.path.isfile(os.path.join(pkg.path, ".piokeep")), + pkg.metadata.spec.owner != "platformio", + pkg.metadata.name not in best_pkg_versions, + pkg.metadata.name in best_pkg_versions + and pkg.metadata.version == best_pkg_versions[pkg.metadata.name], + ] + if not any(skip_conds): + candidates.append(pkg) + + if dry_run: + return candidates + + for pkg in candidates: + pm.uninstall(pkg) + + return candidates diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py new file mode 100644 index 0000000..6babfc9 --- /dev/null +++ b/platformio/package/manager/library.py @@ -0,0 +1,124 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from platformio import util +from platformio.package.exception import MissingPackageManifestError +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageSpec, PackageType +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig + + +class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors + def __init__(self, package_dir=None, **kwargs): + super().__init__( + PackageType.LIBRARY, + package_dir + or ProjectConfig.get_instance().get("platformio", "globallib_dir"), + **kwargs + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.LIBRARY] + + def find_pkg_root(self, path, spec): + try: + return super().find_pkg_root(path, spec) + except MissingPackageManifestError: + pass + assert isinstance(spec, PackageSpec) + + root_dir = self.find_library_root(path) + + # automatically generate library manifest + with open( + os.path.join(root_dir, "library.json"), mode="w", encoding="utf8" + ) as fp: + json.dump( + dict( + name=spec.name, + version=self.generate_rand_version(), + ), + fp, + indent=2, + ) + + return root_dir + + @staticmethod + def find_library_root(path): + root_dir_signs = set(["include", "Include", "inc", "Inc", "src", "Src"]) + root_file_signs = set( + [ + "conanfile.py", # Conan-based library + "CMakeLists.txt", # CMake-based library + ] + ) + for root, dirs, files in os.walk(path): + if not files and len(dirs) == 1: + continue + if set(root_dir_signs) & set(dirs): + return root + if set(root_file_signs) & set(files): + return root + for fname in files: + if fname.endswith((".c", ".cpp", ".h", ".hpp", ".S")): + return root + return path + + def install_dependency(self, dependency): + spec = self.dependency_to_spec(dependency) + # skip built-in dependencies + not_builtin_conds = [spec.external, spec.owner] + if not any(not_builtin_conds): + not_builtin_conds.append(not self.is_builtin_lib(spec.name)) + if any(not_builtin_conds): + return super().install_dependency(dependency) + return None + + @staticmethod + @util.memoized(expire="60s") + def get_builtin_libs(storage_names=None): + # pylint: disable=import-outside-toplevel + from platformio.package.manager.platform import PlatformPackageManager + + items = [] + storage_names = storage_names or [] + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) + for storage in p.get_lib_storages(): + if storage_names and storage["name"] not in storage_names: + continue + lm = LibraryPackageManager(storage["path"]) + items.append( + { + "name": storage["name"], + "path": storage["path"], + "items": lm.legacy_get_installed(), + } + ) + return items + + @classmethod + def is_builtin_lib(cls, name): + for storage in cls.get_builtin_libs(): + for lib in storage["items"]: + if lib.get("name") == name: + return True + return False diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py new file mode 100644 index 0000000..6dbd480 --- /dev/null +++ b/platformio/package/manager/platform.py @@ -0,0 +1,180 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import util +from platformio.http import HTTPClientError, InternetConnectionError +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.core import get_installed_core_packages +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageType +from platformio.platform.exception import IncompatiblePlatform, UnknownBoard +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig + + +class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors + def __init__(self, package_dir=None): + self.config = ProjectConfig.get_instance() + super().__init__( + PackageType.PLATFORM, + package_dir or self.config.get("platformio", "platforms_dir"), + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.PLATFORM] + + def install( # pylint: disable=arguments-differ,too-many-arguments + self, + spec, + skip_dependencies=False, + force=False, + project_env=None, + project_targets=None, + ): + already_installed = self.get_package(spec) + pkg = super().install(spec, force=force, skip_dependencies=True) + try: + p = PlatformFactory.new(pkg) + # set logging level for underlying tool manager + p.pm.set_log_level(self.log.getEffectiveLevel()) + p.ensure_engine_compatible() + except IncompatiblePlatform as exc: + super().uninstall(pkg, skip_dependencies=True) + raise exc + if project_env: + p.configure_project_packages(project_env, project_targets) + if not skip_dependencies: + p.install_required_packages(force=force) + if not already_installed: + p.on_installed() + return pkg + + def uninstall( # pylint: disable=arguments-differ + self, spec, skip_dependencies=False, project_env=None + ): + pkg = self.get_package(spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(spec) + p = PlatformFactory.new(pkg) + # set logging level for underlying tool manager + p.pm.set_log_level(self.log.getEffectiveLevel()) + if project_env: + p.configure_project_packages(project_env) + if not skip_dependencies: + p.uninstall_packages() + assert super().uninstall(pkg, skip_dependencies=True) + p.on_uninstalled() + return pkg + + def update( # pylint: disable=arguments-differ + self, + from_spec, + to_spec=None, + skip_dependencies=False, + project_env=None, + ): + pkg = self.get_package(from_spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(from_spec) + pkg = super().update( + from_spec, + to_spec, + ) + p = PlatformFactory.new(pkg) + # set logging level for underlying tool manager + p.pm.set_log_level(self.log.getEffectiveLevel()) + if project_env: + p.configure_project_packages(project_env) + if not skip_dependencies: + p.update_packages() + return pkg + + @util.memoized(expire="5s") + def get_installed_boards(self): + boards = [] + for pkg in self.get_installed(): + p = PlatformFactory.new(pkg) + for config in p.get_boards().values(): + board = config.get_brief_data() + if board not in boards: + boards.append(board) + return boards + + def get_registered_boards(self): + return self.get_registry_client_instance().fetch_json_data( + "get", "/v2/boards", x_cache_valid="1d" + ) + + def get_all_boards(self): + boards = self.get_installed_boards() + know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] + try: + for board in self.get_registered_boards(): + key = "%s:%s" % (board["platform"], board["id"]) + if key not in know_boards: + boards.append(board) + except (HTTPClientError, InternetConnectionError): + pass + return sorted(boards, key=lambda b: b["name"]) + + def board_config(self, id_, platform=None): + for manifest in self.get_installed_boards(): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): + return manifest + for manifest in self.get_registered_boards(): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): + return manifest + raise UnknownBoard(id_) + + +# +# Helpers +# + + +def remove_unnecessary_platform_packages(dry_run=False): + candidates = [] + required = set() + core_packages = get_installed_core_packages() + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for pkg in p.get_installed_packages(with_optional_versions=True): + required.add(pkg) + + pm = ToolPackageManager() + for pkg in pm.get_installed(): + skip_conds = [ + pkg.metadata.spec.uri, + os.path.isfile(os.path.join(pkg.path, ".piokeep")), + pkg in required, + pkg in core_packages, + ] + if not any(skip_conds): + candidates.append(pkg) + + if dry_run: + return candidates + + for pkg in candidates: + pm.uninstall(pkg) + + return candidates diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py new file mode 100644 index 0000000..0919a33 --- /dev/null +++ b/platformio/package/manager/tool.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageType +from platformio.project.config import ProjectConfig + + +class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors + def __init__(self, package_dir=None): + super().__init__( + PackageType.TOOL, + package_dir + or ProjectConfig.get_instance().get("platformio", "packages_dir"), + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.TOOL] diff --git a/platformio/package/manifest/__init__.py b/platformio/package/manifest/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/package/manifest/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py new file mode 100644 index 0000000..a70af7e --- /dev/null +++ b/platformio/package/manifest/parser.py @@ -0,0 +1,715 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import io +import json +import os +import re +import tarfile +from urllib.parse import urlparse + +from platformio import util +from platformio.compat import get_object_members, string_types +from platformio.http import fetch_remote_content +from platformio.package.exception import ManifestParserError, UnknownManifestError +from platformio.project.helpers import is_platformio_project + + +class ManifestFileType: + PLATFORM_JSON = "platform.json" + LIBRARY_JSON = "library.json" + LIBRARY_PROPERTIES = "library.properties" + MODULE_JSON = "module.json" + PACKAGE_JSON = "package.json" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def from_uri(cls, uri): + for t in sorted(cls.items().values()): + if uri.endswith(t): + return t + return None + + @classmethod + def from_dir(cls, path): + for t in sorted(cls.items().values()): + if os.path.isfile(os.path.join(path, t)): + return t + return None + + +class ManifestParserFactory: + @staticmethod + def read_manifest_contents(path): + last_err = None + for encoding in ("utf-8", "latin-1"): + try: + with io.open(path, encoding=encoding) as fp: + return fp.read() + except UnicodeDecodeError as exc: + last_err = exc + raise last_err + + @classmethod + def new_from_file(cls, path, remote_url=False): + if not path or not os.path.isfile(path): + raise UnknownManifestError("Manifest file does not exist %s" % path) + type_from_uri = ManifestFileType.from_uri(path) + if not type_from_uri: + raise UnknownManifestError("Unknown manifest file type %s" % path) + return ManifestParserFactory.new( + cls.read_manifest_contents(path), type_from_uri, remote_url + ) + + @classmethod + def new_from_dir(cls, path, remote_url=None): + assert os.path.isdir(path), "Invalid directory %s" % path + + type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None + if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)): + return ManifestParserFactory.new( + cls.read_manifest_contents(os.path.join(path, type_from_uri)), + type_from_uri, + remote_url=remote_url, + package_dir=path, + ) + + type_from_dir = ManifestFileType.from_dir(path) + if not type_from_dir: + raise UnknownManifestError( + "Unknown manifest file type in %s directory" % path + ) + return ManifestParserFactory.new( + cls.read_manifest_contents(os.path.join(path, type_from_dir)), + type_from_dir, + remote_url=remote_url, + package_dir=path, + ) + + @staticmethod + def new_from_url(remote_url): + content = fetch_remote_content(remote_url) + return ManifestParserFactory.new( + content, + ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, + remote_url, + ) + + @staticmethod + def new_from_archive(path): + assert path.endswith("tar.gz") + with tarfile.open(path, mode="r:gz") as tf: + for t in sorted(ManifestFileType.items().values()): + for member in (t, "./" + t): + try: + return ManifestParserFactory.new( + tf.extractfile(member).read().decode(), t + ) + except KeyError: + pass + raise UnknownManifestError("Unknown manifest file type in %s archive" % path) + + @staticmethod + def new( # pylint: disable=redefined-builtin + contents, type, remote_url=None, package_dir=None + ): + for _, cls in globals().items(): + if ( + inspect.isclass(cls) + and issubclass(cls, BaseManifestParser) + and cls != BaseManifestParser + and cls.manifest_type == type + ): + return cls(contents, remote_url, package_dir) + raise UnknownManifestError("Unknown manifest file type %s" % type) + + +class BaseManifestParser: + def __init__(self, contents, remote_url=None, package_dir=None): + self.remote_url = remote_url + self.package_dir = package_dir + try: + self._data = self.parse(contents) + except Exception as exc: + raise ManifestParserError("Could not parse manifest -> %s" % exc) from exc + + self._data = self.normalize_repository(self._data) + self._data = self.parse_examples(self._data) + + # remove None fields + for key in list(self._data.keys()): + if self._data[key] is None: + del self._data[key] + + def parse(self, contents): + raise NotImplementedError + + def as_dict(self): + return self._data + + @staticmethod + def str_to_list(value, sep=",", lowercase=False, unique=False): + if isinstance(value, string_types): + value = value.split(sep) + assert isinstance(value, list) + result = [] + for item in value: + item = item.strip() + if not item: + continue + if lowercase: + item = item.lower() + if unique and item in result: + continue + result.append(item) + return result + + @staticmethod + def cleanup_author(author): + assert isinstance(author, dict) + if author.get("email"): + author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) + if "@" not in author["email"]: + author["email"] = None + for key in list(author.keys()): + if author[key] is None: + del author[key] + return author + + @staticmethod + def parse_author_name_and_email(raw): + if raw == "None" or "://" in raw: + return (None, None) + name = raw + email = None + ldel = "<" + rdel = ">" + if ldel in raw and rdel in raw: + name = raw[: raw.index(ldel)] + email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + if "(" in name: + name = name.split("(")[0] + return (name.strip(), email.strip() if email else None) + + @staticmethod + def normalize_repository(data): + url = (data.get("repository") or {}).get("url") + if not url or "://" not in url: + return data + url_attrs = urlparse(url) + if url_attrs.netloc not in ("github.com", "bitbucket.org", "gitlab.com"): + return data + url = "https://%s%s" % (url_attrs.netloc, url_attrs.path) + if url.endswith("/"): + url = url[:-1] + if not url.endswith(".git"): + url += ".git" + data["repository"]["url"] = url + return data + + def parse_examples(self, data): + examples = data.get("examples") + if ( + not examples + or not isinstance(examples, list) + or not all(isinstance(v, dict) for v in examples) + ): + data["examples"] = None + if not data["examples"] and self.package_dir: + data["examples"] = self.parse_examples_from_dir(self.package_dir) + if "examples" in data and not data["examples"]: + del data["examples"] + return data + + @staticmethod + def parse_examples_from_dir(package_dir): + assert os.path.isdir(package_dir) + examples_dir = os.path.join(package_dir, "examples") + if not os.path.isdir(examples_dir): + examples_dir = os.path.join(package_dir, "Examples") + if not os.path.isdir(examples_dir): + return None + + allowed_exts = ( + ".c", + ".cc", + ".cpp", + ".h", + ".hpp", + ".asm", + ".ASM", + ".s", + ".S", + ".ino", + ".pde", + ) + + result = {} + last_pio_project = None + for root, _, files in os.walk(examples_dir): + # skip hidden files, symlinks, and folders + files = [ + f + for f in files + if not f.startswith(".") and not os.path.islink(os.path.join(root, f)) + ] + if os.path.basename(root).startswith(".") or not files: + continue + + if is_platformio_project(root): + last_pio_project = root + result[last_pio_project] = dict( + name=os.path.relpath(root, examples_dir), + base=os.path.relpath(root, package_dir), + files=files, + ) + continue + if last_pio_project: + if root.startswith(last_pio_project): + result[last_pio_project]["files"].extend( + [ + os.path.relpath(os.path.join(root, f), last_pio_project) + for f in files + ] + ) + continue + last_pio_project = None + + matched_files = [f for f in files if f.endswith(allowed_exts)] + if not matched_files: + continue + result[root] = dict( + name="Examples" + if root == examples_dir + else os.path.relpath(root, examples_dir), + base=os.path.relpath(root, package_dir), + files=matched_files, + ) + + result = list(result.values()) + + # normalize example names + for item in result: + item["name"] = item["name"].replace(os.path.sep, "/") + item["name"] = re.sub(r"[^a-z\d\d\-\_/]+", "_", item["name"], flags=re.I) + + return result or None + + +class LibraryJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.LIBRARY_JSON + + def parse(self, contents): + data = json.loads(contents) + data = self._process_renamed_fields(data) + + # normalize Union[str, list] fields + for k in ("keywords", "platforms", "frameworks"): + if k in data: + data[k] = self.str_to_list( + data[k], sep=",", lowercase=True, unique=True + ) + + if "headers" in data: + data["headers"] = self.str_to_list(data["headers"], sep=",", unique=True) + if "authors" in data: + data["authors"] = self._parse_authors(data["authors"]) + if "platforms" in data: + data["platforms"] = self._fix_platforms(data["platforms"]) or None + if "export" in data: + data["export"] = self._parse_export(data["export"]) + if "dependencies" in data: + data["dependencies"] = self._parse_dependencies(data["dependencies"]) + + return data + + @staticmethod + def _process_renamed_fields(data): + if "url" in data: + data["homepage"] = data["url"] + del data["url"] + + for key in ("include", "exclude"): + if key not in data: + continue + if "export" not in data: + data["export"] = {} + data["export"][key] = data[key] + del data[key] + + return data + + def _parse_authors(self, raw): + if not raw: + return None + # normalize Union[dict, list] fields + if not isinstance(raw, list): + raw = [raw] + return [self.cleanup_author(author) for author in raw] + + @staticmethod + def _fix_platforms(items): + assert isinstance(items, list) + if "espressif" in items: + items[items.index("espressif")] = "espressif8266" + return items + + @staticmethod + def _parse_export(raw): + if not isinstance(raw, dict): + return None + result = {} + for k in ("include", "exclude"): + if not raw.get(k): + continue + result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]] + return result + + @staticmethod + def _parse_dependencies(raw): + # compatibility with legacy dependency format + if isinstance(raw, dict) and "name" in raw: + raw = [raw] + + if isinstance(raw, dict): + result = [] + for name, version in raw.items(): + if "/" in name: + owner, name = name.split("/", 1) + result.append(dict(owner=owner, name=name, version=version)) + else: + result.append(dict(name=name, version=version)) + return result + + if isinstance(raw, list): + for i, dependency in enumerate(raw): + if isinstance(dependency, dict): + for k, v in dependency.items(): + if k not in ("platforms", "frameworks", "authors"): + continue + raw[i][k] = util.items_to_list(v) + else: + raw[i] = {"name": dependency} + return raw + raise ManifestParserError( + "Invalid dependencies format, should be list or dictionary" + ) + + +class ModuleJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.MODULE_JSON + + def parse(self, contents): + data = json.loads(contents) + data["frameworks"] = ["mbed"] + data["platforms"] = ["*"] + data["export"] = {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]} + if "author" in data: + data["authors"] = self._parse_authors(data.get("author")) + del data["author"] + if "licenses" in data: + data["license"] = self._parse_license(data.get("licenses")) + del data["licenses"] + if "dependencies" in data: + data["dependencies"] = self._parse_dependencies(data["dependencies"]) + if "keywords" in data: + data["keywords"] = self.str_to_list( + data["keywords"], sep=",", lowercase=True, unique=True + ) + return data + + def _parse_authors(self, raw): + if not raw: + return None + result = [] + for author in raw.split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + result.append(self.cleanup_author(dict(name=name, email=email))) + return result + + @staticmethod + def _parse_license(raw): + if not raw or not isinstance(raw, list): + return None + return raw[0].get("type") + + @staticmethod + def _parse_dependencies(raw): + if isinstance(raw, dict): + return [ + dict(name=name, version=version, frameworks=["mbed"]) + for name, version in raw.items() + ] + raise ManifestParserError("Invalid dependencies format, should be a dictionary") + + +class LibraryPropertiesManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.LIBRARY_PROPERTIES + + def parse(self, contents): + data = self._parse_properties(contents) + repository = self._parse_repository(data) + homepage = data.get("url") or None + if repository and repository["url"] == homepage: + homepage = None + data.update( + dict( + frameworks=["arduino"], + homepage=homepage, + repository=repository or None, + description=self._parse_description(data), + platforms=self._parse_platforms(data) or None, + keywords=self._parse_keywords(data) or None, + export=self._parse_export(), + ) + ) + if "includes" in data: + data["headers"] = self.str_to_list(data["includes"], sep=",", unique=True) + if "author" in data: + data["authors"] = self._parse_authors(data) + for key in ("author", "maintainer"): + if key in data: + del data[key] + if "depends" in data: + data["dependencies"] = self._parse_dependencies(data["depends"]) + return data + + @staticmethod + def _parse_properties(contents): + data = {} + for line in contents.splitlines(): + line = line.strip() + if not line or "=" not in line: + continue + # skip comments + if line.startswith("#"): + continue + key, value = line.split("=", 1) + if not value.strip(): + continue + data[key.strip()] = value.strip() + return data + + @staticmethod + def _parse_description(properties): + lines = [] + for k in ("sentence", "paragraph"): + if k in properties and properties[k] not in lines: + lines.append(properties[k]) + if len(lines) == 2: + if not lines[0].endswith("."): + lines[0] += "." + if len(lines[0]) + len(lines[1]) >= 1000: + del lines[1] + return " ".join(lines) + + def _parse_keywords(self, properties): + return self.str_to_list( + re.split( + r"[\s/]+", + properties.get("category", ""), + ), + lowercase=True, + unique=True, + ) + + def _parse_platforms(self, properties): + result = [] + platforms_map = { + "avr": "atmelavr", + "sam": "atmelsam", + "samd": "atmelsam", + "esp8266": "espressif8266", + "esp32": "espressif32", + "arc32": "intel_arc32", + "stm32": "ststm32", + } + for arch in properties.get("architectures", "").split(","): + if "particle-" in arch: + raise ManifestParserError("Particle is not supported yet") + arch = arch.strip() + if not arch: + continue + if arch == "*": + return ["*"] + if arch in platforms_map: + result.append(platforms_map[arch]) + return self.str_to_list(result, lowercase=True, unique=True) + + def _parse_authors(self, properties): + if "author" not in properties: + return None + authors = [] + for author in properties["author"].split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + authors.append(self.cleanup_author(dict(name=name, email=email))) + for author in properties.get("maintainer", "").split(","): + name, email = self.parse_author_name_and_email(author) + if not name: + continue + found = False + for item in authors: + if item.get("name", "").lower() != name.lower(): + continue + found = True + item["maintainer"] = True + # pylint: disable=unsupported-membership-test + if not item.get("email") and email and "@" in email: + item["email"] = email + if not found: + authors.append( + self.cleanup_author(dict(name=name, email=email, maintainer=True)) + ) + return authors + + def _parse_repository(self, properties): + if self.remote_url: + url_attrs = urlparse(self.remote_url) + repo_path_tokens = url_attrs.path[1:].split("/")[:-1] + if "github" in url_attrs.netloc: + return dict( + type="git", + url="https://github.com/" + "/".join(repo_path_tokens[:2]), + ) + if "raw" in repo_path_tokens: + return dict( + type="git", + url="https://%s/%s" + % ( + url_attrs.netloc, + "/".join(repo_path_tokens[: repo_path_tokens.index("raw")]), + ), + ) + if properties.get("url", "").startswith("https://github.com"): + return dict(type="git", url=properties["url"]) + return None + + def _parse_export(self): + include = None + if self.remote_url: + url_attrs = urlparse(self.remote_url) + repo_path_tokens = url_attrs.path[1:].split("/")[:-1] + if "github" in url_attrs.netloc: + include = "/".join(repo_path_tokens[3:]) or None + elif "raw" in repo_path_tokens: + include = ( + "/".join(repo_path_tokens[repo_path_tokens.index("raw") + 2 :]) + or None + ) + if include: + return dict(include=[include]) + return None + + @staticmethod + def _parse_dependencies(raw): + result = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + if item.endswith(")") and "(" in item: + name, version = item.split("(") + result.append( + dict( + name=name.strip(), + version=version[:-1].strip(), + frameworks=["arduino"], + ) + ) + else: + result.append(dict(name=item, frameworks=["arduino"])) + return result + + +class PlatformJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.PLATFORM_JSON + + def parse(self, contents): + data = json.loads(contents) + if "keywords" in data: + data["keywords"] = self.str_to_list( + data["keywords"], sep=",", lowercase=True, unique=True + ) + if "frameworks" in data: + data["frameworks"] = ( + self.str_to_list( + list(data["frameworks"].keys()), lowercase=True, unique=True + ) + if isinstance(data["frameworks"], dict) + else None + ) + if "packages" in data: + data["dependencies"] = self._parse_dependencies(data["packages"]) + return data + + @staticmethod + def _parse_dependencies(raw): + result = [] + for name, opts in raw.items(): + item = {"name": name} + for k in ("owner", "version"): + if k in opts: + item[k] = opts[k] + result.append(item) + return result + + +class PackageJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.PACKAGE_JSON + + def parse(self, contents): + data = json.loads(contents) + if "keywords" in data: + data["keywords"] = self.str_to_list( + data["keywords"], sep=",", lowercase=True, unique=True + ) + data = self._parse_system(data) + data = self._parse_homepage(data) + data = self._parse_repository(data) + return data + + def _parse_system(self, data): + if "system" not in data: + return data + if data["system"] in ("*", ["*"], "all"): + del data["system"] + return data + data["system"] = self.str_to_list(data["system"], lowercase=True, unique=True) + return data + + @staticmethod + def _parse_homepage(data): + if "url" in data: + data["homepage"] = data["url"] + del data["url"] + return data + + @staticmethod + def _parse_repository(data): + if isinstance(data.get("repository", {}), dict): + return data + data["repository"] = dict(type="git", url=str(data["repository"])) + if data["repository"]["url"].startswith(("github:", "gitlab:", "bitbucket:")): + data["repository"]["url"] = "https://{0}.com/{1}".format( + *(data["repository"]["url"].split(":", 1)) + ) + return data diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py new file mode 100644 index 0000000..95e0810 --- /dev/null +++ b/platformio/package/manifest/schema.py @@ -0,0 +1,284 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-ancestors + +import json +import re + +import marshmallow +import requests +import semantic_version +from marshmallow import Schema, ValidationError, fields, validate, validates + +from platformio.http import fetch_remote_content +from platformio.package.exception import ManifestValidationError +from platformio.util import memoized + + +class BaseSchema(Schema): + class Meta: + unknown = marshmallow.EXCLUDE # pylint: disable=no-member + + def load_manifest(self, data): + return self.load(data) + + def handle_error(self, error, data, **_): # pylint: disable=arguments-differ + raise ManifestValidationError( + error.messages, + data, + error.valid_data if hasattr(error, "valid_data") else error.data, + ) + + +class StrictSchema(BaseSchema): + def handle_error(self, error, data, **_): # pylint: disable=arguments-differ + # skip broken records + if self.many: + error.valid_data = [ + item for idx, item in enumerate(data) if idx not in error.messages + ] + else: + error.valid_data = None + raise error + + +class StrictListField(fields.List): + def _deserialize( # pylint: disable=arguments-differ + self, value, attr, data, **kwargs + ): + try: + return super()._deserialize(value, attr, data, **kwargs) + except ValidationError as exc: + if exc.data: + exc.data = [item for item in exc.data if item is not None] + raise exc + + +class AuthorSchema(StrictSchema): + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + email = fields.Email(validate=validate.Length(min=1, max=50)) + maintainer = fields.Bool(dump_default=False) + url = fields.Url(validate=validate.Length(min=1, max=255)) + + +class RepositorySchema(StrictSchema): + type = fields.Str( + required=True, + validate=validate.OneOf( + ["git", "hg", "svn"], + error="Invalid repository type, please use one of [git, hg, svn]", + ), + ) + url = fields.Str(required=True, validate=validate.Length(min=1, max=255)) + branch = fields.Str(validate=validate.Length(min=1, max=50)) + + +class DependencySchema(StrictSchema): + owner = fields.Str(validate=validate.Length(min=1, max=100)) + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + version = fields.Str(validate=validate.Length(min=1, max=100)) + authors = StrictListField(fields.Str(validate=validate.Length(min=1, max=50))) + platforms = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + frameworks = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + + +class ExportSchema(BaseSchema): + include = StrictListField(fields.Str) + exclude = StrictListField(fields.Str) + + +class ExampleSchema(StrictSchema): + name = fields.Str( + required=True, + validate=[ + validate.Length(min=1, max=255), + validate.Regexp( + r"^[a-zA-Z\d\-\_/\. ]+$", + error="Only [a-zA-Z0-9-_/. ] chars are allowed", + ), + ], + ) + base = fields.Str(required=True) + files = StrictListField(fields.Str, required=True) + + +# Fields + + +class ScriptField(fields.Field): + def _deserialize(self, value, attr, data, **kwargs): + if isinstance(value, (str, list)): + return value + raise ValidationError( + "Script value must be a command (string) or list of arguments" + ) + + +# Scheme + + +class ManifestSchema(BaseSchema): + # Required fields + name = fields.Str( + required=True, + validate=[ + validate.Length(min=1, max=100), + validate.Regexp( + r"^[^:;/,@\<\>]+$", error="The next chars [:;/,@<>] are not allowed" + ), + ], + ) + version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + + # Optional fields + + authors = fields.Nested(AuthorSchema, many=True) + description = fields.Str(validate=validate.Length(min=1, max=1000)) + homepage = fields.Url(validate=validate.Length(min=1, max=255)) + license = fields.Str(validate=validate.Length(min=1, max=255)) + repository = fields.Nested(RepositorySchema) + dependencies = fields.Nested(DependencySchema, many=True) + scripts = fields.Dict( + keys=fields.Str(validate=validate.OneOf(["postinstall", "preuninstall"])), + values=ScriptField(), + ) + + # library.json + export = fields.Nested(ExportSchema) + examples = fields.Nested(ExampleSchema, many=True) + downloadUrl = fields.Url(validate=validate.Length(min=1, max=255)) + + keywords = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^[a-z\d\-_\+\. ]+$", error="Only [a-z0-9+_-. ] chars are allowed" + ), + ] + ) + ) + platforms = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + frameworks = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + headers = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=255), + ] + ) + ) + + # platform.json specific + title = fields.Str(validate=validate.Length(min=1, max=100)) + + # package.json specific + system = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^[a-z\d\-_]+$", error="Only [a-z0-9-_] chars are allowed" + ), + ] + ) + ) + + @validates("version") + def validate_version(self, value): + try: + value = str(value) + assert "." in value + # check leading zeros + try: + semantic_version.Version(value) + except ValueError as exc: + if "Invalid leading zero" in str(exc): + raise exc + semantic_version.Version.coerce(value) + except (AssertionError, ValueError) as exc: + raise ValidationError( + "Invalid semantic versioning format, see https://semver.org/" + ) from exc + + @validates("license") + def validate_license(self, value): + try: + spdx = self.load_spdx_licenses() + except requests.exceptions.RequestException as exc: + raise ValidationError( + "Could not load SPDX licenses for validation" + ) from exc + known_ids = set(item.get("licenseId") for item in spdx.get("licenses", [])) + if value in known_ids: + return True + # parse license expression + # https://spdx.github.io/spdx-spec/SPDX-license-expressions/ + package_ids = [ + item.strip() + for item in re.sub(r"(\s+(?:OR|AND|WITH)\s+|[\(\)])", " ", value).split(" ") + if item.strip() + ] + if known_ids >= set(package_ids): + return True + raise ValidationError( + "Invalid SPDX license identifier. See valid identifiers at " + "https://spdx.org/licenses/" + ) + + @staticmethod + @memoized(expire="1h") + def load_spdx_licenses(): + version = "3.21" + spdx_data_url = ( + "https://raw.githubusercontent.com/spdx/license-list-data/" + f"v{version}/json/licenses.json" + ) + return json.loads(fetch_remote_content(spdx_data_url)) diff --git a/platformio/package/meta.py b/platformio/package/meta.py new file mode 100644 index 0000000..b1c491d --- /dev/null +++ b/platformio/package/meta.py @@ -0,0 +1,528 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import tarfile +from binascii import crc32 +from urllib.parse import urlparse + +import semantic_version + +from platformio import fs +from platformio.compat import get_object_members, hashlib_encode_data, string_types +from platformio.package.manifest.parser import ManifestFileType +from platformio.package.version import SemanticVersionError, cast_version_to_semver +from platformio.util import items_in_list + + +class PackageType: + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r:gz") as tf: + for t in sorted(cls.items().values()): + for manifest in manifest_map[t]: + try: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + + +class PackageCompatibility: + KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors") + + @classmethod + def from_dependency(cls, dependency): + assert isinstance(dependency, dict) + qualifiers = { + key: value + for key, value in dependency.items() + if key in cls.KNOWN_QUALIFIERS + } + return PackageCompatibility(**qualifiers) + + def __init__(self, **kwargs): + self.qualifiers = {} + for key, value in kwargs.items(): + if key not in self.KNOWN_QUALIFIERS: + raise ValueError( + "Unknown package compatibility qualifier -> `%s`" % key + ) + self.qualifiers[key] = value + + def __repr__(self): + return "PackageCompatibility <%s>" % self.qualifiers + + def to_search_qualifiers(self): + return self.qualifiers + + def is_compatible(self, other): + assert isinstance(other, PackageCompatibility) + for key, value in self.qualifiers.items(): + other_value = other.qualifiers.get(key) + if not value or not other_value: + continue + if not items_in_list(value, other_value): + return False + return True + + +class PackageOutdatedResult: + UPDATE_INCREMENT_MAJOR = "major" + UPDATE_INCREMENT_MINOR = "minor" + UPDATE_INCREMENT_PATCH = "patch" + + def __init__(self, current, latest=None, wanted=None, detached=False): + self.current = current + self.latest = latest + self.wanted = wanted + self.detached = detached + + def __repr__(self): + return ( + "PackageOutdatedResult ".format( + current=self.current, + latest=self.latest, + wanted=self.wanted, + detached=self.detached, + ) + ) + + def __setattr__(self, name, value): + if ( + value + and name in ("current", "latest", "wanted") + and not isinstance(value, semantic_version.Version) + ): + value = cast_version_to_semver(str(value)) + return super().__setattr__(name, value) + + @property + def update_increment_type(self): + if not self.current or not self.latest: + return None + patch_conds = [ + self.current.major == self.latest.major, + self.current.minor == self.latest.minor, + ] + if all(patch_conds): + return self.UPDATE_INCREMENT_PATCH + minor_conds = [ + self.current.major == self.latest.major, + self.current.major > 0, + ] + if all(minor_conds): + return self.UPDATE_INCREMENT_MINOR + return self.UPDATE_INCREMENT_MAJOR + + def is_outdated(self, allow_incompatible=False): + if self.detached or not self.latest or self.current == self.latest: + return False + if allow_incompatible: + return self.current != self.latest + if self.wanted: + return self.current != self.wanted + return True + + +class PackageSpec: # pylint: disable=too-many-instance-attributes + def __init__( # pylint: disable=redefined-builtin,too-many-arguments + self, raw=None, owner=None, id=None, name=None, requirements=None, uri=None + ): + self._requirements = None + self.owner = owner + self.id = id + self.name = name + self.uri = uri + self.raw = raw + if requirements: + try: + self.requirements = requirements + except SemanticVersionError as exc: + if not self.name or self.uri or self.raw: + raise exc + self.raw = "%s=%s" % (self.name, requirements) + self._name_is_custom = False + self._parse(self.raw) + + def __eq__(self, other): + return all( + [ + self.owner == other.owner, + self.id == other.id, + self.name == other.name, + self.requirements == other.requirements, + self.uri == other.uri, + ] + ) + + def __hash__(self): + return crc32( + hashlib_encode_data( + "%s-%s-%s-%s-%s" + % (self.owner, self.id, self.name, self.requirements, self.uri) + ) + ) + + def __repr__(self): + return ( + "PackageSpec ".format(**self.as_dict()) + ) + + @property + def external(self): + return bool(self.uri) + + @property + def symlink(self): + return self.uri and self.uri.startswith("symlink://") + + @property + def requirements(self): + return self._requirements + + @requirements.setter + def requirements(self, value): + if not value: + self._requirements = None + return + try: + self._requirements = ( + value + if isinstance(value, semantic_version.SimpleSpec) + else semantic_version.SimpleSpec(str(value)) + ) + except ValueError as exc: + raise SemanticVersionError(exc) from exc + + def humanize(self): + result = "" + if self.uri: + result = self.uri + elif self.name: + if self.owner: + result = self.owner + "/" + result += self.name + elif self.id: + result = "id:%d" % self.id + if self.requirements: + result += " @ " + str(self.requirements) + return result + + def has_custom_name(self): + return self._name_is_custom + + def as_dict(self): + return dict( + owner=self.owner, + id=self.id, + name=self.name, + requirements=str(self.requirements) if self.requirements else None, + uri=self.uri, + ) + + def as_dependency(self): + if self.uri: + return self.raw or self.uri + result = "" + if self.name: + result = "%s/%s" % (self.owner, self.name) if self.owner else self.name + elif self.id: + result = str(self.id) + assert result + if self.requirements: + result = "%s@%s" % (result, self.requirements) + return result + + def _parse(self, raw): + if raw is None: + return + if not isinstance(raw, string_types): + raw = str(raw) + raw = raw.strip() + + parsers = ( + self._parse_local_file, + self._parse_requirements, + self._parse_custom_name, + self._parse_id, + self._parse_owner, + self._parse_uri, + ) + for parser in parsers: + if raw is None: + break + raw = parser(raw) + + # if name is not custom, parse it from URI + if not self.name and self.uri: + self.name = self._parse_name_from_uri(self.uri) + elif raw: + # the leftover is a package name + self.name = raw + + @staticmethod + def _parse_local_file(raw): + if raw.startswith(("file://", "symlink://")) or not any( + c in raw for c in ("/", "\\") + ): + return raw + if os.path.exists(raw): + return "file://%s" % raw + return raw + + def _parse_requirements(self, raw): + if "@" not in raw or raw.startswith(("file://", "symlink://")): + return raw + tokens = raw.rsplit("@", 1) + if any(s in tokens[1] for s in (":", "/")): + return raw + self.requirements = tokens[1].strip() + return tokens[0].strip() + + def _parse_custom_name(self, raw): + if "=" not in raw or raw.startswith("id="): + return raw + tokens = raw.split("=", 1) + if "/" in tokens[0]: + return raw + self.name = tokens[0].strip() + self._name_is_custom = True + return tokens[1].strip() + + def _parse_id(self, raw): + if raw.isdigit(): + self.id = int(raw) + return None + if raw.startswith("id="): + return self._parse_id(raw[3:]) + return raw + + def _parse_owner(self, raw): + if raw.count("/") != 1 or "@" in raw: + return raw + tokens = raw.split("/", 1) + self.owner = tokens[0].strip() + self.name = tokens[1].strip() + return None + + def _parse_uri(self, raw): + if not any(s in raw for s in ("@", ":", "/")): + return raw + self.uri = raw.strip() + parts = urlparse(self.uri) + + # if local file or valid URI with scheme vcs+protocol:// + if ( + parts.scheme in ("file", "symlink://") + or "+" in parts.scheme + or self.uri.startswith("git+") + ): + return None + + # parse VCS + git_conditions = [ + parts.path.endswith(".git"), + # Handle GitHub URL (https://github.com/user/package) + parts.netloc in ("github.com", "gitlab.com", "bitbucket.com") + and not parts.path.endswith((".zip", ".tar.gz")), + ] + hg_conditions = [ + # Handle Developer Mbed URL + # (https://developer.mbed.org/users/user/code/package/) + # (https://os.mbed.com/users/user/code/package/) + parts.netloc + in ("mbed.com", "os.mbed.com", "developer.mbed.org") + ] + if any(git_conditions): + self.uri = "git+" + self.uri + elif any(hg_conditions): + self.uri = "hg+" + self.uri + + return None + + @staticmethod + def _parse_name_from_uri(uri): + if uri.endswith("/"): + uri = uri[:-1] + stop_chars = ["#", "?"] + if uri.startswith(("file://", "symlink://")): + stop_chars.append("@") # detached path + for c in stop_chars: + if c in uri: + uri = uri[: uri.index(c)] + + # parse real repository name from Github + parts = urlparse(uri) + if parts.netloc == "github.com" and parts.path.count("/") > 2: + return parts.path.split("/")[2] + + name = os.path.basename(uri) + if "." in name: + return name.split(".", 1)[0].strip() + return name + + +class PackageMetadata: + def __init__( # pylint: disable=redefined-builtin + self, type, name, version, spec=None + ): + # assert type in PackageType.items().values() + if spec: + assert isinstance(spec, PackageSpec) + self.type = type + self.name = name + self._version = None + self.version = version + self.spec = spec + + def __repr__(self): + return ( + "PackageMetadata +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import shutil +import tarfile +import tempfile + +from platformio import fs +from platformio.compat import IS_WINDOWS +from platformio.package.exception import PackageException, UserSideException +from platformio.package.manifest.parser import ( + LibraryPropertiesManifestParser, + ManifestFileType, + ManifestParserFactory, +) +from platformio.package.manifest.schema import ManifestSchema +from platformio.package.meta import PackageItem +from platformio.package.unpack import FileUnpacker + + +class PackagePacker: + INCLUDE_DEFAULT = list(ManifestFileType.items().values()) + [ + "README", + "README.md", + "README.rst", + "LICENSE", + ] + EXCLUDE_DEFAULT = [ + # PlatformIO internal files + PackageItem.METAFILE_NAME, + ".pio/", + "**/.pio/", + # Hidden files + "._*", + "__*", + ".DS_Store", + ".vscode", + ".cache", + "**/.cache", + "**/__pycache__", + "**/*.pyc", + # VCS + ".git/", + ".hg/", + ".svn/", + ] + EXCLUDE_EXTRA = [ + # Tests + "test", + "tests", + # Docs + "doc", + "docs", + "mkdocs", + "doxygen", + "*.doxyfile", + "html", + "media", + "**/*.[pP][dD][fF]", + "**/*.[dD][oO][cC]", + "**/*.[dD][oO][cC][xX]", + "**/*.[pP][pP][tT]", + "**/*.[pP][pP][tT][xX]", + "**/*.[xX][lL][sS]", + "**/*.[xX][lL][sS][xX]", + "**/*.[dD][oO][xX]", + "**/*.[hH][tT][mM]", + "**/*.[hH][tT][mM][lL]", + "**/*.[tT][eE][xX]", + "**/*.[jJ][sS]", + "**/*.[cC][sS][sS]", + # Binary files + "**/*.[jJ][pP][gG]", + "**/*.[jJ][pP][eE][gG]", + "**/*.[pP][nN][gG]", + "**/*.[gG][iI][fF]", + "**/*.[sS][vV][gG]", + "**/*.[zZ][iI][pP]", + "**/*.[gG][zZ]", + "**/*.3[gG][pP]", + "**/*.[mM][oO][vV]", + "**/*.[mM][pP][34]", + "**/*.[pP][sS][dD]", + "**/*.[wW][aA][wW]", + "**/*.sqlite", + ] + EXCLUDE_LIBRARY_EXTRA = [ + "assets", + "extra", + "extras", + "resources", + "**/build/", + "**/*.flat", + "**/*.[jJ][aA][rR]", + "**/*.[eE][xX][eE]", + "**/*.[bB][iI][nN]", + "**/*.[hH][eE][xX]", + "**/*.[dD][bB]", + "**/*.[dD][aA][tT]", + "**/*.[dD][lL][lL]", + ] + + def __init__(self, package, manifest_uri=None): + self.package = package + self.manifest_uri = manifest_uri + self.manifest_parser = None + + @staticmethod + def get_archive_name(name, version, system=None): + return re.sub( + r"[^\da-zA-Z\-\._\+]+", + "", + "{name}{system}-{version}.tar.gz".format( + name=name, + system=("-" + system) if system else "", + version=version, + ), + ) + + @staticmethod + def load_gitignore_filters(path): + result = [] + with open(path, encoding="utf8") as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith(("#")): + continue + if line.startswith("!"): + result.append(f"+<{line[1:]}>") + else: + result.append(f"-<{line}>") + return result + + def pack(self, dst=None): + tmp_dir = tempfile.mkdtemp() + try: + src = self.package + + # if zip/tar.gz -> unpack to tmp dir + if not os.path.isdir(src): + if IS_WINDOWS: + raise UserSideException( + "Packaging from an archive does not work on Windows OS. Please " + "extract data from `%s` manually and pack a folder instead" + % src + ) + with FileUnpacker(src) as fu: + assert fu.unpack(tmp_dir, silent=True) + src = tmp_dir + + src = self.find_source_root(src) + self.manifest_parser = ManifestParserFactory.new_from_dir(src) + manifest = ManifestSchema().load_manifest(self.manifest_parser.as_dict()) + filename = self.get_archive_name( + manifest["name"], + manifest["version"], + manifest["system"][0] if "system" in manifest else None, + ) + + if not dst: + dst = os.path.join(os.getcwd(), filename) + elif os.path.isdir(dst): + dst = os.path.join(dst, filename) + + return self.create_tarball(src, dst, manifest) + finally: + shutil.rmtree(tmp_dir) + + def find_source_root(self, src): + if self.manifest_uri: + mp = ( + ManifestParserFactory.new_from_file(self.manifest_uri[5:]) + if self.manifest_uri.startswith("file:") + else ManifestParserFactory.new_from_url(self.manifest_uri) + ) + manifest = ManifestSchema().load_manifest(mp.as_dict()) + include = manifest.get("export", {}).get("include", []) + if len(include) == 1: + if not os.path.isdir(os.path.join(src, include[0])): + raise PackageException( + "Non existing `include` directory `%s` in a package" + % include[0] + ) + return os.path.join(src, include[0]) + + for root, _, __ in os.walk(src): + if ManifestFileType.from_dir(root): + return root + + return src + + def create_tarball(self, src, dst, manifest): + include = manifest.get("export", {}).get("include") + exclude = manifest.get("export", {}).get("exclude") + # remap root + if ( + include + and len(include) == 1 + and os.path.isdir(os.path.join(src, include[0])) + ): + src = os.path.join(src, include[0]) + with open( + os.path.join(src, "library.json"), mode="w", encoding="utf8" + ) as fp: + manifest_updated = manifest.copy() + del manifest_updated["export"]["include"] + json.dump(manifest_updated, fp, indent=2, ensure_ascii=False) + include = None + + src_filters = self.compute_src_filters(src, include, exclude) + with tarfile.open(dst, "w:gz") as tar: + for f in fs.match_src_files(src, src_filters, followlinks=False): + tar.add(os.path.join(src, f), f) + return dst + + def compute_src_filters(self, src, include, exclude): + exclude_extra = self.EXCLUDE_EXTRA[:] + # extend with library extra filters + if any( + os.path.isfile(os.path.join(src, name)) + for name in ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ) + ): + exclude_extra.extend(self.EXCLUDE_LIBRARY_EXTRA) + + result = ["+<%s>" % p for p in include or ["*", ".*"]] + result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT] + # exclude items declared in manifest + result += ["-<%s>" % p for p in exclude or []] + + # apply extra excludes if no custom "export" field in manifest + if (not include and not exclude) or isinstance( + self.manifest_parser, LibraryPropertiesManifestParser + ): + result += ["-<%s>" % p for p in exclude_extra] + if os.path.exists(os.path.join(src, ".gitignore")): + result += self.load_gitignore_filters(os.path.join(src, ".gitignore")) + + # always include manifests and relevant files + result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT] + return result diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py new file mode 100644 index 0000000..9d8919c --- /dev/null +++ b/platformio/package/unpack.py @@ -0,0 +1,209 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from tarfile import open as tarfile_open +from time import mktime +from zipfile import ZipFile + +import click + +from platformio import fs +from platformio.compat import is_terminal +from platformio.package.exception import PackageException + + +class ExtractArchiveItemError(PackageException): + MESSAGE = ( + "Could not extract `{0}` to `{1}`. Try to disable antivirus " + "tool or check this solution -> https://bit.ly/faq-package-manager" + ) + + +class BaseArchiver: + def __init__(self, arhfileobj): + self._afo = arhfileobj + + def get_items(self): + raise NotImplementedError() + + def get_item_filename(self, item): + raise NotImplementedError() + + def is_link(self, item): + raise NotImplementedError() + + def extract_item(self, item, dest_dir): + self._afo.extract(item, dest_dir) + self.after_extract(item, dest_dir) + + def after_extract(self, item, dest_dir): + pass + + def close(self): + self._afo.close() + + +class TARArchiver(BaseArchiver): + def __init__(self, archpath): + super().__init__(tarfile_open(archpath)) # pylint: disable=consider-using-with + + def get_items(self): + return self._afo.getmembers() + + def get_item_filename(self, item): + return item.name + + @staticmethod + def is_link(item): # pylint: disable=arguments-differ + return item.islnk() or item.issym() + + @staticmethod + def resolve_path(path): + return os.path.realpath(os.path.abspath(path)) + + def is_bad_path(self, path, base): + return not self.resolve_path(os.path.join(base, path)).startswith(base) + + def is_bad_link(self, item, base): + return not self.resolve_path( + os.path.join(os.path.join(base, os.path.dirname(item.name)), item.linkname) + ).startswith(base) + + def extract_item(self, item, dest_dir): + dest_dir = self.resolve_path(dest_dir) + bad_conds = [ + self.is_bad_path(item.name, dest_dir), + self.is_link(item) and self.is_bad_link(item, dest_dir), + ] + if not any(bad_conds): + super().extract_item(item, dest_dir) + else: + click.secho( + "Blocked insecure item `%s` from TAR archive" % item.name, + fg="red", + err=True, + ) + + +class ZIPArchiver(BaseArchiver): + def __init__(self, archpath): + super().__init__(ZipFile(archpath)) # pylint: disable=consider-using-with + + @staticmethod + def preserve_permissions(item, dest_dir): + attrs = item.external_attr >> 16 + if attrs: + os.chmod(os.path.join(dest_dir, item.filename), attrs) + + @staticmethod + def preserve_mtime(item, dest_dir): + fs.change_filemtime( + os.path.join(dest_dir, item.filename), + mktime(tuple(item.date_time) + tuple([0, 0, 0])), + ) + + @staticmethod + def is_link(_): # pylint: disable=arguments-differ + return False + + def get_items(self): + return self._afo.infolist() + + def get_item_filename(self, item): + return item.filename + + def after_extract(self, item, dest_dir): + self.preserve_permissions(item, dest_dir) + self.preserve_mtime(item, dest_dir) + + +class FileUnpacker: + def __init__(self, path): + self.path = path + self._archiver = None + + def __enter__(self): + self._archiver = self.new_archiver(self.path) + return self + + def __exit__(self, *args): + if self._archiver: + self._archiver.close() + + @staticmethod + def new_archiver(path): + magic_map = { + b"\x1f\x8b\x08": TARArchiver, + b"\x42\x5a\x68": TARArchiver, + b"\x50\x4b\x03\x04": ZIPArchiver, + } + magic_len = max(len(k) for k in magic_map) + with open(path, "rb") as fp: + data = fp.read(magic_len) + for magic, archiver in magic_map.items(): + if data.startswith(magic): + return archiver(path) + raise PackageException("Unknown archive type '%s'" % path) + + def unpack( + self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False + ): # pylint: disable=too-many-branches + assert self._archiver + label = "Unpacking" + items = self._archiver.get_items() + if not dest_dir: + dest_dir = os.getcwd() + + if not with_progress or silent: + if not silent: + click.echo(f"{label}...") + for item in items: + self._archiver.extract_item(item, dest_dir) + elif not is_terminal(): + click.echo(f"{label} 0%", nl=False) + print_percent_step = 10 + printed_percents = 0 + unpacked_nums = 0 + for item in items: + self._archiver.extract_item(item, dest_dir) + unpacked_nums += 1 + if (unpacked_nums / len(items) * 100) >= ( + printed_percents + print_percent_step + ): + printed_percents += print_percent_step + click.echo(f" {printed_percents}%", nl=False) + click.echo("") + else: + with click.progressbar( + items, + label=label, + update_min_steps=min(50, len(items) / 100), # every 50 files or less + ) as pb: + for item in pb: + self._archiver.extract_item(item, dest_dir) + + if not check_unpacked: + return True + + # check on disk + for item in self._archiver.get_items(): + filename = self._archiver.get_item_filename(item) + item_path = os.path.join(dest_dir, filename) + try: + if not self._archiver.is_link(item) and not os.path.exists(item_path): + raise ExtractArchiveItemError(filename, dest_dir) + except NotImplementedError: + pass + return True diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py new file mode 100644 index 0000000..6734839 --- /dev/null +++ b/platformio/package/vcsclient.py @@ -0,0 +1,273 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import subprocess +from urllib.parse import urlparse + +from platformio import proc +from platformio.exception import UserSideException + + +class VCSBaseException(UserSideException): + pass + + +class VCSClientFactory: + @staticmethod + def new(src_dir, remote_url, silent=False): + result = urlparse(remote_url) + type_ = result.scheme + tag = None + if not type_ and remote_url.startswith("git+"): + type_ = "git" + remote_url = remote_url[4:] + elif "+" in result.scheme: + type_, _ = result.scheme.split("+", 1) + remote_url = remote_url[len(type_) + 1 :] + if "#" in remote_url: + remote_url, tag = remote_url.rsplit("#", 1) + if not type_: + raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) + try: + obj = globals()["%sClient" % type_.capitalize()]( + src_dir, remote_url, tag, silent + ) + assert isinstance(obj, VCSClientBase) + return obj + except (KeyError, AssertionError) as exc: + raise VCSBaseException( + "VCS: Unknown repository type %s" % remote_url + ) from exc + + +class VCSClientBase: + command = None + + def __init__(self, src_dir, remote_url=None, tag=None, silent=False): + self.src_dir = src_dir + self.remote_url = remote_url + self.tag = tag + self.silent = silent + self.check_client() + + def check_client(self): + try: + assert self.command + if self.silent: + self.get_cmd_output(["--version"]) + else: + assert self.run_cmd(["--version"]) + except (AssertionError, OSError) as exc: + raise VCSBaseException( + "VCS: `%s` client is not installed in your system" % self.command + ) from exc + return True + + @property + def storage_dir(self): + return os.path.join(self.src_dir, "." + self.command) + + def export(self): + raise NotImplementedError + + def update(self): + raise NotImplementedError + + @property + def can_be_updated(self): + return not self.tag + + def get_current_revision(self): + raise NotImplementedError + + def get_latest_revision(self): + return None if self.can_be_updated else self.get_current_revision() + + def run_cmd(self, args, **kwargs): + args = [self.command] + args + if "cwd" not in kwargs: + kwargs["cwd"] = self.src_dir + if "env" not in kwargs: + kwargs["env"] = os.environ + try: + subprocess.check_call(args, **kwargs) + return True + except subprocess.CalledProcessError as exc: + raise VCSBaseException( + "VCS: Could not process command %s" % exc.cmd + ) from exc + + def get_cmd_output(self, args, **kwargs): + args = [self.command] + args + if "cwd" not in kwargs: + kwargs["cwd"] = self.src_dir + result = proc.exec_command(args, **kwargs) + if result["returncode"] == 0: + return result["out"].strip() + raise VCSBaseException( + "VCS: Could not receive an output from `%s` command (%s)" % (args, result) + ) + + +class GitClient(VCSClientBase): + command = "git" + _configured = False + + def __init__(self, *args, **kwargs): + self.configure() + super().__init__(*args, **kwargs) + + @classmethod + def configure(cls): + if cls._configured: + return True + cls._configured = True + try: + result = proc.exec_command([cls.command, "--exec-path"]) + if result["returncode"] != 0: + return False + path = result["out"].strip() + if path: + proc.append_env_path("PATH", path) + return True + except (subprocess.CalledProcessError, FileNotFoundError): + pass + return False + + def check_client(self): + try: + return VCSClientBase.check_client(self) + except UserSideException as exc: + raise UserSideException( + "Please install Git client from https://git-scm.com/downloads" + ) from exc + + def get_branches(self): + output = self.get_cmd_output(["branch"]) + output = output.replace("*", "") # fix active branch + return [b.strip() for b in output.split("\n")] + + def get_current_branch(self): + output = self.get_cmd_output(["branch"]) + for line in output.split("\n"): + line = line.strip() + if line.startswith("*"): + branch = line[1:].strip() + if branch != "(no branch)": + return branch + return None + + def get_tags(self): + output = self.get_cmd_output(["tag", "-l"]) + return [t.strip() for t in output.split("\n")] + + @staticmethod + def is_commit_id(text): + return text and re.match(r"[0-9a-f]{7,}$", text) is not None + + @property + def can_be_updated(self): + return not self.tag or not self.is_commit_id(self.tag) + + def export(self): + is_commit = self.is_commit_id(self.tag) + args = ["clone", "--recursive"] + if not self.tag or not is_commit: + args += ["--depth", "1"] + if self.tag: + args += ["--branch", self.tag] + args += [self.remote_url, self.src_dir] + assert self.run_cmd(args, cwd=os.getcwd()) + if is_commit: + assert self.run_cmd(["reset", "--hard", self.tag]) + return self.run_cmd( + ["submodule", "update", "--init", "--recursive", "--force"] + ) + return True + + def update(self): + args = ["pull", "--recurse-submodules"] + return self.run_cmd(args) + + def get_current_revision(self): + return self.get_cmd_output(["rev-parse", "--short", "HEAD"]) + + def get_latest_revision(self): + if not self.can_be_updated: + return self.get_current_revision() + branch = self.get_current_branch() + if not branch: + return None + + branch_ref = f"refs/heads/{branch}" + result = self.get_cmd_output(["ls-remote", self.remote_url, branch_ref]) + if not result: + return None + + for line in result.split("\n"): + sha, ref = line.strip().split("\t") + if ref == branch_ref: + return sha[:7] + + return None + + +class HgClient(VCSClientBase): + command = "hg" + + def export(self): + args = ["clone"] + if self.tag: + args.extend(["--updaterev", self.tag]) + args.extend([self.remote_url, self.src_dir]) + return self.run_cmd(args) + + def update(self): + args = ["pull", "--update"] + return self.run_cmd(args) + + def get_current_revision(self): + return self.get_cmd_output(["identify", "--id"]) + + def get_latest_revision(self): + if not self.can_be_updated: + return self.get_latest_revision() + return self.get_cmd_output(["identify", "--id", self.remote_url]) + + +class SvnClient(VCSClientBase): + command = "svn" + + def export(self): + args = ["checkout"] + if self.tag: + args.extend(["--revision", self.tag]) + args.extend([self.remote_url, self.src_dir]) + return self.run_cmd(args) + + def update(self): + args = ["update"] + return self.run_cmd(args) + + def get_current_revision(self): + output = self.get_cmd_output( + ["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"] + ) + for line in output.split("\n"): + line = line.strip() + if line.startswith("Revision:"): + return line.split(":", 1)[1].strip() + raise VCSBaseException("Could not detect current SVN revision") diff --git a/platformio/package/version.py b/platformio/package/version.py new file mode 100644 index 0000000..909d83e --- /dev/null +++ b/platformio/package/version.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import semantic_version + +from platformio.exception import UserSideException + + +class SemanticVersionError(UserSideException): + pass + + +def cast_version_to_semver(value, force=True, raise_exception=False): + assert value + try: + return semantic_version.Version(value) + except ValueError: + pass + if force: + try: + return semantic_version.Version.coerce(value) + except ValueError: + pass + if raise_exception: + raise SemanticVersionError("Invalid SemVer version %s" % value) + # parse commit hash + if re.match(r"^[\da-f]+$", value, flags=re.I): + return semantic_version.Version("0.0.0+sha." + value) + return semantic_version.Version("0.0.0+" + value) + + +def pepver_to_semver(pepver): + return cast_version_to_semver( + re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) + ) + + +def get_original_version(version): + if version.count(".") != 2: + return None + _, raw = version.split(".")[:2] + if int(raw) <= 99: + return None + if int(raw) <= 9999: + return "%s.%s" % (raw[:-2], int(raw[-2:])) + return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) diff --git a/platformio/platform/__init__.py b/platformio/platform/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/platform/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py new file mode 100644 index 0000000..2a3aa49 --- /dev/null +++ b/platformio/platform/_packages.py @@ -0,0 +1,92 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.meta import PackageSpec + + +class PlatformPackagesMixin: + def get_package_spec(self, name, version=None): + return PackageSpec( + owner=self.packages[name].get("owner"), + name=name, + requirements=version or self.packages[name].get("version"), + ) + + def get_package(self, name, spec=None): + if not name: + return None + return self.pm.get_package(spec or self.get_package_spec(name)) + + def get_package_dir(self, name): + pkg = self.get_package(name) + return pkg.path if pkg else None + + def get_package_version(self, name): + pkg = self.get_package(name) + return str(pkg.metadata.version) if pkg else None + + def get_installed_packages(self, with_optional=True, with_optional_versions=False): + result = [] + for name, options in dict(sorted(self.packages.items())).items(): + if not with_optional and options.get("optional"): + continue + versions = [options.get("version")] + if with_optional_versions: + versions.extend(options.get("optionalVersions", [])) + for version in versions: + if not version: + continue + pkg = self.get_package(name, self.get_package_spec(name, version)) + if pkg: + result.append(pkg) + return result + + def dump_used_packages(self): + result = [] + for name, options in self.packages.items(): + if options.get("optional"): + continue + pkg = self.get_package(name) + if not pkg or not pkg.metadata: + continue + item = {"name": pkg.metadata.name, "version": str(pkg.metadata.version)} + if pkg.metadata.spec.external: + item["src_url"] = pkg.metadata.spec.uri + result.append(item) + return result + + def install_package(self, name, spec=None, force=False): + return self.pm.install(spec or self.get_package_spec(name), force=force) + + def install_required_packages(self, force=False): + for name, options in self.packages.items(): + if options.get("optional"): + continue + self.install_package(name, force=force) + + def uninstall_packages(self): + for pkg in self.get_installed_packages(): + self.pm.uninstall(pkg) + + def update_packages(self): + for pkg in self.get_installed_packages(): + self.pm.update(pkg, to_spec=self.get_package_spec(pkg.metadata.name)) + + def are_outdated_packages(self): + for pkg in self.get_installed_packages(): + if self.pm.outdated( + pkg, self.get_package_spec(pkg.metadata.name) + ).is_outdated(allow_incompatible=False): + return True + return False diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py new file mode 100644 index 0000000..949a452 --- /dev/null +++ b/platformio/platform/_run.py @@ -0,0 +1,183 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import os +import re +import sys +from urllib.parse import quote + +import click + +from platformio import app, fs, proc, telemetry +from platformio.compat import hashlib_encode_data +from platformio.package.manager.core import get_core_package_dir +from platformio.platform.exception import BuildScriptNotFound +from platformio.run.helpers import KNOWN_CLEAN_TARGETS, KNOWN_FULLCLEAN_TARGETS + + +class PlatformRunMixin: + LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) + + @staticmethod + def encode_scons_arg(value): + if isinstance(value, (list, tuple, dict)): + value = json.dumps(value) + return base64.urlsafe_b64encode(hashlib_encode_data(value)).decode() + + @staticmethod + def decode_scons_arg(data): + value = base64.urlsafe_b64decode(data).decode() + if value.startswith(("[", "{")): + value = json.loads(value) + return value + + def run( # pylint: disable=too-many-arguments + self, variables, targets, silent, verbose, jobs + ): + assert isinstance(variables, dict) + assert isinstance(targets, list) + + self.ensure_engine_compatible() + + self.silent = silent + self.verbose = verbose or app.get_setting("force_verbose") + + if "build_script" not in variables: + variables["build_script"] = self.get_build_script() + if not os.path.isfile(variables["build_script"]): + raise BuildScriptNotFound(variables["build_script"]) + + telemetry.log_platform_run(self, self.config, variables["pioenv"], targets) + result = self._run_scons(variables, targets, jobs) + + assert "returncode" in result + + return result + + def _run_scons(self, variables, targets, jobs): + scons_dir = get_core_package_dir("tool-scons") + args = [ + proc.get_pythonexe_path(), + os.path.join(scons_dir, "scons.py"), + "-Q", + "--warn=no-no-parallel-support", + "--jobs", + str(jobs), + "--sconstruct", + os.path.join(fs.get_source_dir(), "builder", "main.py"), + ] + args.append("PIOVERBOSE=%d" % int(self.verbose)) + # pylint: disable=protected-access + args.append("ISATTY=%d" % int(click._compat.isatty(sys.stdout))) + # encode and append variables + for key, value in variables.items(): + args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) + + if set(KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS) & set(targets): + args.append("--clean") + args.append( + "FULLCLEAN=%d" + % (1 if set(KNOWN_FULLCLEAN_TARGETS) & set(targets) else 0) + ) + elif targets: + args.extend(targets) + + # force SCons output to Unicode + os.environ["PYTHONIOENCODING"] = "utf-8" + + if targets and "menuconfig" in targets: + return proc.exec_command( + args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin + ) + + if click._compat.isatty(sys.stdout): + + def _write_and_flush(stream, data): + try: + stream.write(data) + stream.flush() + except IOError: + pass + + return proc.exec_command( + args, + stdout=proc.BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: None + if self.silent + else _write_and_flush(sys.stdout, data), + ), + stderr=proc.BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data), + ), + ) + + return proc.exec_command( + args, + stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), + stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + ) + + def _on_stdout_line(self, line): + if "`buildprog' is up to date." in line: + return + self._echo_line(line, level=1) + + def _on_stderr_line(self, line): + is_error = self.LINE_ERROR_RE.search(line) is not None + self._echo_line(line, level=3 if is_error else 2) + + a_pos = line.find("fatal error:") + b_pos = line.rfind(": No such file or directory") + if a_pos == -1 or b_pos == -1: + return + self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) + + def _echo_line(self, line, level): + if line.startswith("scons: "): + line = line[7:] + assert 1 <= level <= 3 + if self.silent and (level < 2 or not line): + return + fg = (None, "yellow", "red")[level - 1] + if level == 1 and "is up to date" in line: + fg = "green" + click.secho(line, fg=fg, err=level > 1, nl=False) + + @staticmethod + def _echo_missed_dependency(filename): + if "/" in filename or not filename.endswith((".h", ".hpp")): + return + banner = """ +{dots} +* Looking for {filename_styled} dependency? Check our library registry! +* +* CLI > platformio lib search "header:{filename}" +* Web > {link} +* +{dots} +""".format( + filename=filename, + filename_styled=click.style(filename, fg="cyan"), + link=click.style( + "https://registry.platformio.org/search?q=header:%s" + % quote(filename, safe=""), + fg="blue", + ), + dots="*" * (56 + len(filename)), + ) + click.echo(banner, err=True) diff --git a/platformio/platform/base.py b/platformio/platform/base.py new file mode 100644 index 0000000..51b0221 --- /dev/null +++ b/platformio/platform/base.py @@ -0,0 +1,239 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import semantic_version + +from platformio import __version__, fs +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.version import pepver_to_semver +from platformio.platform._packages import PlatformPackagesMixin +from platformio.platform._run import PlatformRunMixin +from platformio.platform.board import PlatformBoardConfig +from platformio.platform.exception import IncompatiblePlatform, UnknownBoard +from platformio.project.config import ProjectConfig + + +class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods + PlatformPackagesMixin, PlatformRunMixin +): + CORE_SEMVER = pepver_to_semver(__version__) + _BOARDS_CACHE = {} + + def __init__(self, manifest_path): + self.manifest_path = manifest_path + self.project_env = None # set by factory.from_env(env) + self.silent = False + self.verbose = False + + self._manifest = fs.load_json(manifest_path) + self._BOARDS_CACHE = {} + self._custom_packages = None + + self.config = ProjectConfig.get_instance() + self.pm = ToolPackageManager(self.config.get("platformio", "packages_dir")) + + @property + def name(self): + return self._manifest["name"] + + @property + def title(self): + return self._manifest["title"] + + @property + def description(self): + return self._manifest["description"] + + @property + def version(self): + return self._manifest["version"] + + @property + def homepage(self): + return self._manifest.get("homepage") + + @property + def repository_url(self): + return self._manifest.get("repository", {}).get("url") + + @property + def license(self): + return self._manifest.get("license") + + @property + def frameworks(self): + return self._manifest.get("frameworks") + + @property + def engines(self): + return self._manifest.get("engines") + + @property + def manifest(self): + return self._manifest + + @property + def packages(self): + packages = self._manifest.get("packages", {}) + for item in self._custom_packages or []: + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 1) + spec = self.pm.ensure_spec(name) + options = {"version": version.strip(), "optional": False} + if spec.owner: + options["owner"] = spec.owner + if spec.name not in packages: + packages[spec.name] = {} + packages[spec.name].update(**options) + return packages + + def ensure_engine_compatible(self): + if not self.engines or "platformio" not in self.engines: + return True + core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) + if self.CORE_SEMVER in core_spec: + return True + # PIO Core 6 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 + if any( + semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3, 4, 5) + ): + return True + raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) + + def get_dir(self): + return os.path.dirname(self.manifest_path) + + def get_build_script(self): + main_script = os.path.join(self.get_dir(), "builder", "main.py") + if os.path.isfile(main_script): + return main_script + raise NotImplementedError() + + def is_embedded(self): + for opts in self.packages.values(): + if opts.get("type") == "uploader": + return True + return False + + def get_boards(self, id_=None): + def _append_board(board_id, manifest_path): + config = PlatformBoardConfig(manifest_path) + if "platform" in config and config.get("platform") != self.name: + return + if "platforms" in config and self.name not in config.get("platforms"): + return + config.manifest["platform"] = self.name + self._BOARDS_CACHE[board_id] = config + + bdirs = [ + self.config.get("platformio", "boards_dir"), + os.path.join(self.config.get("platformio", "core_dir"), "boards"), + os.path.join(self.get_dir(), "boards"), + ] + + if id_ is None: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + for item in sorted(os.listdir(boards_dir)): + _id = item[:-5] + if not item.endswith(".json") or _id in self._BOARDS_CACHE: + continue + _append_board(_id, os.path.join(boards_dir, item)) + else: + if id_ not in self._BOARDS_CACHE: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + manifest_path = os.path.join(boards_dir, "%s.json" % id_) + if os.path.isfile(manifest_path): + _append_board(id_, manifest_path) + break + if id_ not in self._BOARDS_CACHE: + raise UnknownBoard(id_) + return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE + + def board_config(self, id_): + return self.get_boards(id_) + + def get_package_type(self, name): + return self.packages[name].get("type") + + def configure_project_packages(self, env, targets=None): + options = self.config.items(env=env, as_dict=True) + if "framework" in options: + # support PIO Core 3.0 dev/platforms + options["pioframework"] = options["framework"] + # override user custom packages + self._custom_packages = options.get("platform_packages") + self.configure_default_packages(options, targets or []) + + def configure_default_packages(self, options, targets): + # enable used frameworks + for framework in options.get("framework", []): + if not self.frameworks: + continue + framework = framework.lower().strip() + if not framework or framework not in self.frameworks: + continue + _pkg_name = self.frameworks[framework].get("package") + if _pkg_name: + self.packages[_pkg_name]["optional"] = False + + # enable upload tools for upload targets + if any(["upload" in t for t in targets] + ["program" in targets]): + for name, opts in self.packages.items(): + if opts.get("type") == "uploader": + self.packages[name]["optional"] = False + # skip all packages in "nobuild" mode + # allow only upload tools and frameworks + elif "nobuild" in targets and opts.get("type") != "framework": + self.packages[name]["optional"] = True + + def configure_debug_session(self, debug_config): + raise NotImplementedError + + def generate_sample_code(self, project_config, environment): + raise NotImplementedError + + def on_installed(self): + pass + + def on_uninstalled(self): + pass + + def get_lib_storages(self): + storages = {} + for opts in (self.frameworks or {}).values(): + if "package" not in opts: + continue + pkg = self.get_package(opts["package"]) + if not pkg or not os.path.isdir(os.path.join(pkg.path, "libraries")): + continue + libs_dir = os.path.join(pkg.path, "libraries") + storages[libs_dir] = opts["package"] + libcores_dir = os.path.join(libs_dir, "__cores__") + if not os.path.isdir(libcores_dir): + continue + for item in os.listdir(libcores_dir): + libcore_dir = os.path.join(libcores_dir, item) + if not os.path.isdir(libcore_dir): + continue + storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) + + return [dict(name=name, path=path) for path, name in storages.items()] diff --git a/platformio/platform/board.py b/platformio/platform/board.py new file mode 100644 index 0000000..78d4894 --- /dev/null +++ b/platformio/platform/board.py @@ -0,0 +1,145 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs, util +from platformio.compat import MISSING +from platformio.debug.exception import DebugInvalidOptionsError, DebugSupportError +from platformio.exception import InvalidJSONFile, UserSideException +from platformio.platform.exception import InvalidBoardManifest + + +class PlatformBoardConfig: + def __init__(self, manifest_path): + self._id = os.path.basename(manifest_path)[:-5] + assert os.path.isfile(manifest_path) + self.manifest_path = manifest_path + try: + self._manifest = fs.load_json(manifest_path) + except InvalidJSONFile as exc: + raise InvalidBoardManifest(manifest_path) from exc + if not set(["name", "url", "vendor"]) <= set(self._manifest): + raise UserSideException( + "Please specify name, url and vendor fields for " + manifest_path + ) + + def get(self, path, default=MISSING): + try: + value = self._manifest + for k in path.split("."): + value = value[k] + return value + except KeyError: + if default != MISSING: + return default + raise KeyError("Invalid board option '%s'" % path) + + def update(self, path, value): + newdict = None + for key in path.split(".")[::-1]: + if newdict is None: + newdict = {key: value} + else: + newdict = {key: newdict} + util.merge_dicts(self._manifest, newdict) + + def __contains__(self, key): + try: + self.get(key) + return True + except KeyError: + return False + + @property + def id(self): + return self._id + + @property + def id_(self): + return self.id + + @property + def manifest(self): + return self._manifest + + def get_brief_data(self): + result = { + "id": self.id, + "name": self._manifest["name"], + "platform": self._manifest.get("platform"), + "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), + "fcpu": int( + "".join( + [ + c + for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) + if c.isdigit() + ] + ) + ), + "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), + "rom": self._manifest.get("upload", {}).get("maximum_size", 0), + "frameworks": self._manifest.get("frameworks"), + "vendor": self._manifest["vendor"], + "url": self._manifest["url"], + } + if self._manifest.get("connectivity"): + result["connectivity"] = self._manifest.get("connectivity") + debug = self.get_debug_data() + if debug: + result["debug"] = debug + return result + + def get_debug_data(self): + if not self._manifest.get("debug", {}).get("tools"): + return None + tools = {} + for name, options in self._manifest["debug"]["tools"].items(): + tools[name] = {} + for key, value in options.items(): + if key in ("default", "onboard") and value: + tools[name][key] = value + return {"tools": tools} + + def get_debug_tool_name(self, custom=None): + debug_tools = self._manifest.get("debug", {}).get("tools") + tool_name = custom + if tool_name == "custom": + return tool_name + if not debug_tools: + raise DebugSupportError(self._manifest["name"]) + if tool_name: + if tool_name in debug_tools: + return tool_name + raise DebugInvalidOptionsError( + "Unknown debug tool `%s`. Please use one of `%s` or `custom`" + % (tool_name, ", ".join(sorted(list(debug_tools)))) + ) + + # automatically select best tool + data = {"default": [], "onboard": [], "external": []} + for key, value in debug_tools.items(): + if value.get("default"): + data["default"].append(key) + elif value.get("onboard"): + data["onboard"].append(key) + data["external"].append(key) + + for key, value in data.items(): + if not value: + continue + return sorted(value)[0] + + assert any(item for item in data) diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py new file mode 100644 index 0000000..18ace0d --- /dev/null +++ b/platformio/platform/exception.py @@ -0,0 +1,46 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import UserSideException + + +class PlatformException(UserSideException): + pass + + +class UnknownPlatform(PlatformException): + MESSAGE = "Unknown development platform '{0}'" + + +class IncompatiblePlatform(PlatformException): + MESSAGE = ( + "Development platform '{0}' is not compatible with PlatformIO Core v{1} and " + "depends on PlatformIO Core {2}.\n" + ) + + +class UnknownBoard(PlatformException): + MESSAGE = "Unknown board ID '{0}'" + + +class InvalidBoardManifest(PlatformException): + MESSAGE = "Invalid board JSON manifest '{0}'" + + +class UnknownFramework(PlatformException): + MESSAGE = "Unknown framework '{0}'" + + +class BuildScriptNotFound(PlatformException): + MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py new file mode 100644 index 0000000..334888b --- /dev/null +++ b/platformio/platform/factory.py @@ -0,0 +1,103 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import sys + +from platformio import fs +from platformio.compat import load_python_module +from platformio.package.meta import PackageItem +from platformio.platform import base +from platformio.platform.exception import UnknownPlatform +from platformio.project.config import ProjectConfig +from platformio.project.exception import UndefinedEnvPlatformError + + +class PlatformFactory: + @staticmethod + def get_clsname(name): + name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) + return "%sPlatform" % name.lower().capitalize() + + @staticmethod + def load_platform_module(name, path): + # backward compatibiility with the legacy dev-platforms + sys.modules["platformio.managers.platform"] = base + try: + return load_python_module("platformio.platform.%s" % name, path) + except ImportError as exc: + raise UnknownPlatform(name) from exc + + @classmethod + def new(cls, pkg_or_spec, autoinstall=False) -> base.PlatformBase: + # pylint: disable=import-outside-toplevel + from platformio.package.manager.platform import PlatformPackageManager + + platform_dir = None + platform_name = None + if isinstance(pkg_or_spec, PackageItem): + platform_dir = pkg_or_spec.path + platform_name = pkg_or_spec.metadata.name + elif isinstance(pkg_or_spec, (str, bytes)) and os.path.isdir(pkg_or_spec): + platform_dir = pkg_or_spec + else: + pkg = PlatformPackageManager().get_package(pkg_or_spec) + if pkg: + platform_dir = pkg.path + platform_name = pkg.metadata.name + + if not platform_dir or not os.path.isfile( + os.path.join(platform_dir, "platform.json") + ): + if autoinstall: + return cls.new( + PlatformPackageManager().install( + pkg_or_spec, skip_dependencies=True + ) + ) + raise UnknownPlatform(pkg_or_spec) + + if not platform_name: + platform_name = fs.load_json(os.path.join(platform_dir, "platform.json"))[ + "name" + ] + + platform_cls = None + if os.path.isfile(os.path.join(platform_dir, "platform.py")): + platform_cls = getattr( + cls.load_platform_module( + platform_name, os.path.join(platform_dir, "platform.py") + ), + cls.get_clsname(platform_name), + ) + else: + platform_cls = type( + str(cls.get_clsname(platform_name)), (base.PlatformBase,), {} + ) + + _instance = platform_cls(os.path.join(platform_dir, "platform.json")) + assert isinstance(_instance, base.PlatformBase) + return _instance + + @classmethod + def from_env(cls, env, targets=None, autoinstall=False): + config = ProjectConfig.get_instance() + spec = config.get(f"env:{env}", "platform", None) + if not spec: + raise UndefinedEnvPlatformError(env) + p = cls.new(spec, autoinstall=autoinstall) + p.project_env = env + p.configure_project_packages(env, targets) + return p diff --git a/platformio/proc.py b/platformio/proc.py new file mode 100644 index 0000000..acfce4d --- /dev/null +++ b/platformio/proc.py @@ -0,0 +1,219 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +from contextlib import contextmanager +from threading import Thread + +from platformio import exception +from platformio.compat import ( + IS_WINDOWS, + get_filesystem_encoding, + get_locale_encoding, + string_types, +) + + +class AsyncPipeBase: + def __init__(self): + self._fd_read, self._fd_write = os.pipe() + self._pipe_reader = os.fdopen( + self._fd_read, encoding="utf-8", errors="backslashreplace" + ) + self._buffer = "" + self._thread = Thread(target=self.run) + self._thread.start() + + def get_buffer(self): + return self._buffer + + def fileno(self): + return self._fd_write + + def run(self): + try: + self.do_reading() + except (KeyboardInterrupt, SystemExit, IOError): + self.close() + + def do_reading(self): + raise NotImplementedError() + + def close(self): + self._buffer = "" + os.close(self._fd_write) + self._thread.join() + + +class BuildAsyncPipe(AsyncPipeBase): + def __init__(self, line_callback, data_callback): + self.line_callback = line_callback + self.data_callback = data_callback + super().__init__() + + def do_reading(self): + line = "" + print_immediately = False + + for char in iter(lambda: self._pipe_reader.read(1), ""): + self._buffer += char + + if line and char.strip() and line[-3:] == (char * 3): + print_immediately = True + + if print_immediately: + # leftover bytes + if line: + self.data_callback(line) + line = "" + self.data_callback(char) + if char == "\n": + print_immediately = False + else: + line += char + if char != "\n": + continue + self.line_callback(line) + line = "" + + self._pipe_reader.close() + + +class LineBufferedAsyncPipe(AsyncPipeBase): + def __init__(self, line_callback): + self.line_callback = line_callback + super().__init__() + + def do_reading(self): + for line in iter(self._pipe_reader.readline, ""): + self._buffer += line + self.line_callback(line) + self._pipe_reader.close() + + +def exec_command(*args, **kwargs): + result = {"out": None, "err": None, "returncode": None} + + default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) + default.update(kwargs) + kwargs = default + + with subprocess.Popen(*args, **kwargs) as p: + try: + result["out"], result["err"] = p.communicate() + result["returncode"] = p.returncode + except KeyboardInterrupt as exc: + raise exception.AbortedByUser() from exc + finally: + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipeBase): + kwargs[s].close() # pylint: disable=no-member + + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipeBase): + result[s[3:]] = kwargs[s].get_buffer() # pylint: disable=no-member + + for key, value in result.items(): + if isinstance(value, bytes): + try: + result[key] = value.decode( + get_locale_encoding() or get_filesystem_encoding() + ) + except UnicodeDecodeError: + result[key] = value.decode("latin-1") + if value and isinstance(value, string_types): + result[key] = value.strip() + + return result + + +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + +def is_ci(): + return os.getenv("CI", "").lower() == "true" + + +def is_container(): + if os.path.exists("/.dockerenv"): + return True + if not os.path.isfile("/proc/1/cgroup"): + return False + with open("/proc/1/cgroup", encoding="utf8") as fp: + return ":/docker/" in fp.read() + + +def get_pythonexe_path(): + return os.environ.get("PYTHONEXEPATH", os.path.normpath(sys.executable)) + + +def copy_pythonpath_to_osenv(): + _PYTHONPATH = [] + if "PYTHONPATH" in os.environ: + _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) + for p in os.sys.path: + conditions = [p not in _PYTHONPATH] + if not IS_WINDOWS: + conditions.append( + os.path.isdir(os.path.join(p, "click")) + or os.path.isdir(os.path.join(p, "platformio")) + ) + if all(conditions): + _PYTHONPATH.append(p) + os.environ["PYTHONPATH"] = os.pathsep.join(_PYTHONPATH) + + +def where_is_program(program, envpath=None): + env = os.environ.copy() + if envpath: + env["PATH"] = envpath + + # look up in $PATH + for bin_dir in env.get("PATH", "").split(os.pathsep): + if os.path.isfile(os.path.join(bin_dir, program)): + return os.path.join(bin_dir, program) + if IS_WINDOWS and os.path.isfile(os.path.join(bin_dir, "%s.exe" % program)): + return os.path.join(bin_dir, "%s.exe" % program) + + # try OS's built-in commands + try: + result = exec_command(["where" if IS_WINDOWS else "which", program], env=env) + if result["returncode"] == 0 and os.path.isfile(result["out"].strip()): + return result["out"].strip() + except OSError: + pass + + return program + + +def append_env_path(name, value): + cur_value = os.environ.get(name) or "" + if cur_value and value in cur_value.split(os.pathsep): + return cur_value + os.environ[name] = os.pathsep.join([cur_value, value]) + return os.environ[name] + + +def force_exit(code=0): + os._exit(code) # pylint: disable=protected-access diff --git a/platformio/project/__init__.py b/platformio/project/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/project/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/project/cli.py b/platformio/project/cli.py new file mode 100644 index 0000000..534bb72 --- /dev/null +++ b/platformio/project/cli.py @@ -0,0 +1,32 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.project.commands.config import project_config_cmd +from platformio.project.commands.init import project_init_cmd +from platformio.project.commands.metadata import project_metadata_cmd + + +@click.group( + "project", + commands=[ + project_config_cmd, + project_init_cmd, + project_metadata_cmd, + ], + short_help="Project Manager", +) +def cli(): + pass diff --git a/platformio/project/commands/__init__.py b/platformio/project/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/project/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/project/commands/config.py b/platformio/project/commands/config.py new file mode 100644 index 0000000..5e4ed21 --- /dev/null +++ b/platformio/project/commands/config.py @@ -0,0 +1,104 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError +from platformio.project.helpers import is_platformio_project + + +@click.command("config", short_help="Show computed configuration") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option("--lint", is_flag=True) +@click.option("--json-output", is_flag=True) +def project_config_cmd(project_dir, lint, json_output): + if not is_platformio_project(project_dir): + raise NotPlatformIOProjectError(project_dir) + with fs.cd(project_dir): + if lint: + return lint_configuration(json_output) + return print_configuration(json_output) + + +def print_configuration(json_output=False): + config = ProjectConfig.get_instance() + if json_output: + return click.echo(config.to_json()) + click.echo( + "Computed project configuration for %s" % click.style(os.getcwd(), fg="cyan") + ) + for section, options in config.as_tuple(): + click.secho(section, fg="cyan") + click.echo("-" * len(section)) + click.echo( + tabulate( + [ + (name, "=", "\n".join(value) if isinstance(value, list) else value) + for name, value in options + ], + tablefmt="plain", + ) + ) + click.echo() + return None + + +def lint_configuration(json_output=False): + result = ProjectConfig.lint() + errors = result["errors"] + warnings = result["warnings"] + if json_output: + return click.echo(result) + if not errors and not warnings: + return click.secho( + 'The "platformio.ini" configuration file is free from linting errors.', + fg="green", + ) + if errors: + click.echo( + tabulate( + [ + ( + click.style(error["type"], fg="red"), + error["message"], + error.get("source", "") + (f":{error.get('lineno')}") + if "lineno" in error + else "", + ) + for error in errors + ], + tablefmt="plain", + ) + ) + if warnings: + click.echo( + tabulate( + [ + (click.style("Warning", fg="yellow"), warning) + for warning in warnings + ], + tablefmt="plain", + ) + ) + return None diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py new file mode 100644 index 0000000..51ae46a --- /dev/null +++ b/platformio/project/commands/init.py @@ -0,0 +1,408 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long,too-many-arguments,too-many-locals + + +import json +import os + +import click + +from platformio import fs +from platformio.package.commands.install import install_project_dependencies +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.exception import UndefinedEnvPlatformError +from platformio.project.helpers import is_platformio_project +from platformio.project.integration.generator import ProjectGenerator +from platformio.project.options import ProjectOptions + + +def validate_boards(ctx, param, value): # pylint: disable=unused-argument + pm = PlatformPackageManager() + for id_ in value: + try: + pm.board_config(id_) + except UnknownBoard as exc: + raise click.BadParameter( + "`%s`. Please search for board ID using `platformio boards` " + "command" % id_ + ) from exc + return value + + +@click.command("init", short_help="Initialize a project or update existing") +@click.option( + "--project-dir", + "-d", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), +) +@click.option( + "-b", "--board", "boards", multiple=True, metavar="ID", callback=validate_boards +) +@click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) +@click.option("-e", "--environment", help="Update existing environment") +@click.option( + "-O", + "--project-option", + "project_options", + multiple=True, + help="A `name=value` pair", +) +@click.option("--sample-code", is_flag=True) +@click.option("--no-install-dependencies", is_flag=True) +@click.option("--env-prefix", default="") +@click.option("-s", "--silent", is_flag=True) +def project_init_cmd( + project_dir, + boards, + ide, + environment, + project_options, + sample_code, + no_install_dependencies, + env_prefix, + silent, +): + is_new_project = not is_platformio_project(project_dir) + if is_new_project: + if not silent: + print_header(project_dir) + init_base_project(project_dir) + + with fs.cd(project_dir): + if environment: + update_project_env(environment, project_options) + elif boards: + update_board_envs(project_dir, boards, project_options, env_prefix) + + generator = None + config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + if ide: + config.validate() + # init generator and pick the best env if user didn't specify + generator = ProjectGenerator(config, environment, ide, boards) + if not environment: + environment = generator.env_name + + # resolve project dependencies + if not no_install_dependencies and (environment or boards): + install_project_dependencies( + options=dict( + project_dir=project_dir, + environments=[environment] if environment else [], + silent=silent, + ) + ) + + if environment and sample_code: + init_sample_code(config, environment) + + if generator: + if not silent: + click.echo( + "Updating metadata for the %s IDE..." % click.style(ide, fg="cyan") + ) + generator.generate() + + if is_new_project: + init_cvs_ignore() + + if not silent: + print_footer(is_new_project) + + +def print_header(project_dir): + click.echo("The following files/directories have been created in ", nl=False) + try: + click.secho(project_dir, fg="cyan") + except UnicodeEncodeError: + click.secho(json.dumps(project_dir), fg="cyan") + click.echo("%s - Put project header files here" % click.style("include", fg="cyan")) + click.echo( + "%s - Put project specific (private) libraries here" + % click.style("lib", fg="cyan") + ) + click.echo("%s - Put project source files here" % click.style("src", fg="cyan")) + click.echo( + "%s - Project Configuration File" % click.style("platformio.ini", fg="cyan") + ) + + +def print_footer(is_new_project): + action = "initialized" if is_new_project else "updated" + return click.secho( + f"Project has been successfully {action}!", + fg="green", + ) + + +def init_base_project(project_dir): + with fs.cd(project_dir): + config = ProjectConfig() + config.save() + dir_to_readme = [ + (config.get("platformio", "src_dir"), None), + (config.get("platformio", "include_dir"), init_include_readme), + (config.get("platformio", "lib_dir"), init_lib_readme), + (config.get("platformio", "test_dir"), init_test_readme), + ] + for path, cb in dir_to_readme: + if os.path.isdir(path): + continue + os.makedirs(path) + if cb: + cb(path) + + +def init_include_readme(include_dir): + with open(os.path.join(include_dir, "README"), mode="w", encoding="utf8") as fp: + fp.write( + """ +This directory is intended for project header files. + +A header file is a file containing C declarations and macro definitions +to be shared between several project source files. You request the use of a +header file in your project source file (C, C++, etc) located in `src` folder +by including it, with the C preprocessing directive `#include'. + +```src/main.c + +#include "header.h" + +int main (void) +{ + ... +} +``` + +Including a header file produces the same results as copying the header file +into each source file that needs it. Such copying would be time-consuming +and error-prone. With a header file, the related declarations appear +in only one place. If they need to be changed, they can be changed in one +place, and programs that include the header file will automatically use the +new version when next recompiled. The header file eliminates the labor of +finding and changing all the copies as well as the risk that a failure to +find one copy will result in inconsistencies within a program. + +In C, the usual convention is to give header files names that end with `.h'. +It is most portable to use only letters, digits, dashes, and underscores in +header file names, and at most one dot. + +Read more about using header files in official GCC documentation: + +* Include Syntax +* Include Operation +* Once-Only Headers +* Computed Includes + +https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html +""", + ) + + +def init_lib_readme(lib_dir): + with open(os.path.join(lib_dir, "README"), mode="w", encoding="utf8") as fp: + fp.write( + """ +This directory is intended for project specific (private) libraries. +PlatformIO will compile them to static libraries and link into executable file. + +The source code of each library should be placed in a an own separate directory +("lib/your_library_name/[here are source files]"). + +For example, see a structure of the following two libraries `Foo` and `Bar`: + +|--lib +| | +| |--Bar +| | |--docs +| | |--examples +| | |--src +| | |- Bar.c +| | |- Bar.h +| | |- library.json (optional, custom build options, etc) https://docs.platformio.org/page/librarymanager/config.html +| | +| |--Foo +| | |- Foo.c +| | |- Foo.h +| | +| |- README --> THIS FILE +| +|- platformio.ini +|--src + |- main.c + +and a contents of `src/main.c`: +``` +#include +#include + +int main (void) +{ + ... +} + +``` + +PlatformIO Library Dependency Finder will find automatically dependent +libraries scanning project source files. + +More information about PlatformIO Library Dependency Finder +- https://docs.platformio.org/page/librarymanager/ldf.html +""", + ) + + +def init_test_readme(test_dir): + with open(os.path.join(test_dir, "README"), mode="w", encoding="utf8") as fp: + fp.write( + """ +This directory is intended for PlatformIO Test Runner and project tests. + +Unit Testing is a software testing method by which individual units of +source code, sets of one or more MCU program modules together with associated +control data, usage procedures, and operating procedures, are tested to +determine whether they are fit for use. Unit testing finds problems early +in the development cycle. + +More information about PlatformIO Unit Testing: +- https://docs.platformio.org/en/latest/advanced/unit-testing/index.html +""", + ) + + +def init_cvs_ignore(): + conf_path = ".gitignore" + if os.path.isfile(conf_path): + return + with open(conf_path, mode="w", encoding="utf8") as fp: + fp.write(".pio\n") + + +def update_board_envs(project_dir, boards, extra_project_options, env_prefix): + config = ProjectConfig( + os.path.join(project_dir, "platformio.ini"), parse_extra=False + ) + used_boards = [] + for section in config.sections(): + cond = [section.startswith("env:"), config.has_option(section, "board")] + if all(cond): + used_boards.append(config.get(section, "board")) + + pm = PlatformPackageManager() + modified = False + for id_ in boards: + board_config = pm.board_config(id_) + if id_ in used_boards: + continue + used_boards.append(id_) + modified = True + + envopts = {"platform": board_config["platform"], "board": id_} + # find default framework for board + frameworks = board_config.get("frameworks") + if frameworks: + envopts["framework"] = frameworks[0] + + for item in extra_project_options: + if "=" not in item: + continue + _name, _value = item.split("=", 1) + envopts[_name.strip()] = _value.strip() + + section = "env:%s%s" % (env_prefix, id_) + config.add_section(section) + + for option, value in envopts.items(): + config.set(section, option, value) + + if modified: + config.save() + + +def update_project_env(environment, extra_project_options=None): + if not extra_project_options: + return + env_section = "env:%s" % environment + option_to_sections = {"platformio": [], env_section: []} + for item in extra_project_options: + assert "=" in item + name, value = item.split("=", 1) + name = name.strip() + destination = env_section + for option in ProjectOptions.values(): + if option.scope in option_to_sections and option.name == name: + destination = option.scope + break + option_to_sections[destination].append((name, value.strip())) + + config = ProjectConfig( + "platformio.ini", parse_extra=False, expand_interpolations=False + ) + for section, options in option_to_sections.items(): + if not options: + continue + if not config.has_section(section): + config.add_section(section) + for name, value in options: + config.set(section, name, value) + + config.save() + + +def init_sample_code(config, environment): + try: + p = PlatformFactory.from_env(environment) + return p.generate_sample_code(config, environment) + except (NotImplementedError, UndefinedEnvPlatformError): + pass + + framework = config.get(f"env:{environment}", "framework", None) + if framework != ["arduino"]: + return None + main_content = """ +#include + +// put function declarations here: +int myFunction(int, int); + +void setup() { + // put your setup code here, to run once: + int result = myFunction(2, 3); +} + +void loop() { + // put your main code here, to run repeatedly: +} + +// put function definitions here: +int myFunction(int x, int y) { + return x + y; +} +""" + is_cpp_project = p.name not in ["intel_mcs51", "ststm8"] + src_dir = config.get("platformio", "src_dir") + main_path = os.path.join(src_dir, "main.%s" % ("cpp" if is_cpp_project else "c")) + if os.path.isfile(main_path): + return None + if not os.path.isdir(src_dir): + os.makedirs(src_dir) + with open(main_path, mode="w", encoding="utf8") as fp: + fp.write(main_content.strip()) + return True diff --git a/platformio/project/commands/metadata.py b/platformio/project/commands/metadata.py new file mode 100644 index 0000000..ee9f5b1 --- /dev/null +++ b/platformio/project/commands/metadata.py @@ -0,0 +1,81 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.package.commands.install import install_project_dependencies +from platformio.project.config import ProjectConfig +from platformio.project.helpers import load_build_metadata + + +@click.command( + "metadata", short_help="Dump metadata intended for IDE extensions/plugins" +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option("-e", "--environment", "environments", multiple=True) +@click.option("--json-output", is_flag=True) +@click.option("--json-output-path", type=click.Path()) +def project_metadata_cmd(project_dir, environments, json_output, json_output_path): + project_dir = os.path.abspath(project_dir) + with fs.cd(project_dir): + config = ProjectConfig.get_instance() + config.validate(environments) + environments = list(environments or config.envs()) + build_metadata = load_build_metadata(project_dir, environments) + + if not json_output: + install_project_dependencies( + options=dict( + project_dir=project_dir, + environments=environments, + ) + ) + click.echo() + + if json_output or json_output_path: + if json_output_path: + if os.path.isdir(json_output_path): + json_output_path = os.path.join(json_output_path, "metadata.json") + with open(json_output_path, mode="w", encoding="utf8") as fp: + json.dump(build_metadata, fp) + click.secho(f"Saved metadata to the {json_output_path}", fg="green") + if json_output: + click.echo(json.dumps(build_metadata)) + return + + for envname, metadata in build_metadata.items(): + click.echo("Environment: " + click.style(envname, fg="cyan", bold=True)) + click.echo("=" * (13 + len(envname))) + click.echo( + tabulate( + [ + (click.style(name, bold=True), "=", json.dumps(value, indent=2)) + for name, value in metadata.items() + ], + tablefmt="plain", + ) + ) + click.echo() + + return diff --git a/platformio/project/config.py b/platformio/project/config.py new file mode 100644 index 0000000..8020fdd --- /dev/null +++ b/platformio/project/config.py @@ -0,0 +1,524 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import configparser +import glob +import json +import os +import re + +import click + +from platformio import fs +from platformio.compat import MISSING, string_types +from platformio.project import exception +from platformio.project.options import ProjectOptions + +CONFIG_HEADER = """ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; https://docs.platformio.org/page/projectconf.html +""" + + +class ProjectConfigBase: + ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I) + INLINE_COMMENT_RE = re.compile(r"\s+;.*$") + VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") + + CUSTOM_OPTION_PREFIXES = ("custom_", "board_") + + expand_interpolations = True + warnings = [] + + _parser = None + _parsed = [] + + @staticmethod + def parse_multi_values(items): + result = [] + if not items: + return result + if not isinstance(items, (list, tuple)): + items = items.split("\n" if "\n" in items else ", ") + for item in items: + item = item.strip() + # comment + if not item or item.startswith((";", "#")): + continue + if ";" in item: + item = ProjectConfigBase.INLINE_COMMENT_RE.sub("", item).strip() + result.append(item) + return result + + @staticmethod + def get_default_path(): + from platformio import app # pylint: disable=import-outside-toplevel + + return app.get_session_var("custom_project_conf") or os.path.join( + os.getcwd(), "platformio.ini" + ) + + def __init__(self, path=None, parse_extra=True, expand_interpolations=True): + path = self.get_default_path() if path is None else path + self.path = path + self.expand_interpolations = expand_interpolations + self.warnings = [] + self._parsed = [] + self._parser = configparser.ConfigParser(inline_comment_prefixes=("#", ";")) + if path and os.path.isfile(path): + self.read(path, parse_extra) + + self._maintain_renamed_options() + + def __getattr__(self, name): + return getattr(self._parser, name) + + def read(self, path, parse_extra=True): + if path in self._parsed: + return + self._parsed.append(path) + try: + self._parser.read(path, "utf-8") + except configparser.Error as exc: + raise exception.InvalidProjectConfError(path, str(exc)) from exc + + if not parse_extra: + return + + # load extra configs + for pattern in self.get("platformio", "extra_configs", []): + if pattern.startswith("~"): + pattern = fs.expanduser(pattern) + for item in glob.glob(pattern, recursive=True): + self.read(item) + + def _maintain_renamed_options(self): + renamed_options = {} + for option in ProjectOptions.values(): + if option.oldnames: + renamed_options.update({name: option.name for name in option.oldnames}) + + for section in self._parser.sections(): + scope = self.get_section_scope(section) + if scope not in ("platformio", "env"): + continue + for option in self._parser.options(section): + if option in renamed_options: + self.warnings.append( + "`%s` configuration option in section [%s] is " + "deprecated and will be removed in the next release! " + "Please use `%s` instead" + % (option, section, renamed_options[option]) + ) + # # rename on-the-fly + # self._parser.set( + # section, + # renamed_options[option], + # self._parser.get(section, option), + # ) + # self._parser.remove_option(section, option) + continue + + # unknown + unknown_conditions = [ + ("%s.%s" % (scope, option)) not in ProjectOptions, + scope != "env" + or not option.startswith(self.CUSTOM_OPTION_PREFIXES), + ] + if all(unknown_conditions): + self.warnings.append( + "Ignore unknown configuration option `%s` " + "in section [%s]" % (option, section) + ) + return True + + @staticmethod + def get_section_scope(section): + return section.split(":", 1)[0] if ":" in section else section + + def walk_options(self, root_section): + extends_queue = ( + ["env", root_section] if root_section.startswith("env:") else [root_section] + ) + extends_done = [] + while extends_queue: + section = extends_queue.pop() + extends_done.append(section) + if not self._parser.has_section(section): + continue + for option in self._parser.options(section): + yield (section, option) + if self._parser.has_option(section, "extends"): + extends_queue.extend( + self.parse_multi_values(self._parser.get(section, "extends")) + ) + + def options(self, section=None, env=None): + result = [] + assert section or env + if not section: + section = "env:" + env + + if not self.expand_interpolations: + return self._parser.options(section) + + for _, option in self.walk_options(section): + if option not in result: + result.append(option) + + # handle system environment variables + scope = self.get_section_scope(section) + for option_meta in ProjectOptions.values(): + if option_meta.scope != scope or option_meta.name in result: + continue + if option_meta.sysenvvar and option_meta.sysenvvar in os.environ: + result.append(option_meta.name) + + return result + + def has_option(self, section, option): + if self._parser.has_option(section, option): + return True + return option in self.options(section) + + def items(self, section=None, env=None, as_dict=False): + assert section or env + if not section: + section = "env:" + env + if as_dict: + return { + option: self.get(section, option) for option in self.options(section) + } + return [(option, self.get(section, option)) for option in self.options(section)] + + def set(self, section, option, value): + if value is None: + value = "" + if isinstance(value, (list, tuple)): + value = "\n".join(value) + elif isinstance(value, bool): + value = "yes" if value else "no" + elif isinstance(value, (int, float)): + value = str(value) + # start multi-line value from a new line + if "\n" in value and not value.startswith("\n"): + value = "\n" + value + self._parser.set(section, option, value) + + def resolve_renamed_option(self, section, old_name): + scope = self.get_section_scope(section) + if scope not in ("platformio", "env"): + return None + for option_meta in ProjectOptions.values(): + if ( + option_meta.oldnames + and option_meta.scope == scope + and old_name in option_meta.oldnames + ): + return option_meta.name + return None + + def find_option_meta(self, section, option): + scope = self.get_section_scope(section) + if scope not in ("platformio", "env"): + return None + option_meta = ProjectOptions.get("%s.%s" % (scope, option)) + if option_meta: + return option_meta + for option_meta in ProjectOptions.values(): + if option_meta.scope == scope and option in (option_meta.oldnames or []): + return option_meta + return None + + def _traverse_for_value(self, section, option, option_meta=None): + for _section, _option in self.walk_options(section): + if _option == option or ( + option_meta + and ( + option_meta.name == _option + or _option in (option_meta.oldnames or []) + ) + ): + return self._parser.get(_section, _option) + return MISSING + + def getraw( + self, section, option, default=MISSING + ): # pylint: disable=too-many-branches + if not self.expand_interpolations: + return self._parser.get(section, option) + + option_meta = self.find_option_meta(section, option) + value = self._traverse_for_value(section, option, option_meta) + + if not option_meta: + if value == MISSING: + value = ( + default if default != MISSING else self._parser.get(section, option) + ) + return self._expand_interpolations(section, value) + + if option_meta.sysenvvar: + envvar_value = os.getenv(option_meta.sysenvvar) + if not envvar_value and option_meta.oldnames: + for oldoption in option_meta.oldnames: + envvar_value = os.getenv("PLATFORMIO_" + oldoption.upper()) + if envvar_value: + break + if envvar_value and option_meta.multiple: + if value == MISSING: + value = "" + value += ("\n" if value else "") + envvar_value + elif envvar_value: + value = envvar_value + + if value == MISSING: + value = default if default != MISSING else option_meta.default + if callable(value): + value = value() + if value == MISSING: + return None + + return self._expand_interpolations(section, value) + + def _expand_interpolations(self, parent_section, value): + if ( + not value + or not isinstance(value, string_types) + or not all(["${" in value, "}" in value]) + ): + return value + return self.VARTPL_RE.sub( + lambda match: self._re_interpolation_handler(parent_section, match), value + ) + + def _re_interpolation_handler(self, parent_section, match): + section, option = match.group(1), match.group(2) + # handle system environment variables + if section == "sysenv": + return os.getenv(option) + # handle ${this.*} + if section == "this": + section = parent_section + if option == "__env__": + if not parent_section.startswith("env:"): + raise exception.ProjectOptionValueError( + f"`${{this.__env__}}` is called from the `{parent_section}` " + "section that is not valid PlatformIO environment, see", + option, + " ", + section, + ) + return parent_section[4:] + # handle nested calls + try: + value = self.get(section, option) + except RecursionError as exc: + raise exception.ProjectOptionValueError( + "Infinite recursion has been detected", + option, + " ", + section, + ) from exc + if isinstance(value, list): + return "\n".join(value) + return str(value) + + def get(self, section, option, default=MISSING): + value = None + try: + value = self.getraw(section, option, default) + except configparser.Error as exc: + raise exception.InvalidProjectConfError(self.path, str(exc)) + + option_meta = self.find_option_meta(section, option) + if not option_meta: + return value + + if option_meta.validate: + value = option_meta.validate(value) + if option_meta.multiple: + value = self.parse_multi_values(value or []) + try: + return self.cast_to(value, option_meta.type) + except click.BadParameter as exc: + if not self.expand_interpolations: + return value + raise exception.ProjectOptionValueError( + exc.format_message(), + option, + " (%s) " % option_meta.description, + section, + ) + + @staticmethod + def cast_to(value, to_type): + items = value + if not isinstance(value, (list, tuple)): + items = [value] + items = [ + to_type(item) if isinstance(to_type, click.ParamType) else item + for item in items + ] + return items if isinstance(value, (list, tuple)) else items[0] + + def envs(self): + return [s[4:] for s in self._parser.sections() if s.startswith("env:")] + + def default_envs(self): + return self.get("platformio", "default_envs", []) + + def get_default_env(self): + default_envs = self.default_envs() + if default_envs: + return default_envs[0] + envs = self.envs() + return envs[0] if envs else None + + def validate(self, envs=None, silent=False): + if not os.path.isfile(self.path): + raise exception.NotPlatformIOProjectError(os.path.dirname(self.path)) + + known_envs = set(self.envs()) + + # check envs + if not known_envs: + raise exception.ProjectEnvsNotAvailableError() + unknown_envs = set(list(envs or []) + self.default_envs()) - known_envs + if unknown_envs: + raise exception.UnknownEnvNamesError( + ", ".join(unknown_envs), ", ".join(known_envs) + ) + + for env in known_envs: + # check envs names + if not self.ENVNAME_RE.match(env): + raise exception.InvalidEnvNameError(env) + + # check simultaneous use of `monitor_raw` and `monitor_filters` + if self.get(f"env:{env}", "monitor_raw", False) and self.get( + f"env:{env}", "monitor_filters", None + ): + self.warnings.append( + "The `monitor_raw` and `monitor_filters` options cannot be " + f"used simultaneously for the `{env}` environment in the " + "`platformio.ini` file. The `monitor_filters` option will " + "be disabled to avoid conflicts." + ) + + if not silent: + for warning in self.warnings: + click.secho("Warning! %s" % warning, fg="yellow") + + return True + + +class ProjectConfigLintMixin: + @classmethod + def lint(cls, path=None): + errors = [] + warnings = [] + try: + config = cls.get_instance(path) + config.validate(silent=True) + warnings = config.warnings + config.as_tuple() + except Exception as exc: # pylint: disable=broad-exception-caught + if exc.__cause__ is not None: + exc = exc.__cause__ + + item = {"type": exc.__class__.__name__, "message": str(exc)} + for attr in ("lineno", "source"): + if hasattr(exc, attr): + item[attr] = getattr(exc, attr) + + if item["type"] == "ParsingError" and hasattr(exc, "errors"): + for lineno, line in getattr(exc, "errors"): + errors.append( + { + "type": item["type"], + "message": f"Parsing error: {line}", + "lineno": lineno, + "source": item["source"], + } + ) + else: + errors.append(item) + return {"errors": errors, "warnings": warnings} + + +class ProjectConfigDirsMixin: + def get_optional_dir(self, name): + """ + Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs + PlatformIO IDE for Atom depends on platformio-node-helpers@~7.2.0 + PIO Home 3.0 Project Inspection depends on it + """ + return self.get("platformio", f"{name}_dir") + + +class ProjectConfig(ProjectConfigBase, ProjectConfigLintMixin, ProjectConfigDirsMixin): + _instances = {} + + @staticmethod + def get_instance(path=None): + path = ProjectConfig.get_default_path() if path is None else path + mtime = os.path.getmtime(path) if os.path.isfile(path) else 0 + instance = ProjectConfig._instances.get(path) + if instance and instance["mtime"] != mtime: + instance = None + if not instance: + instance = {"mtime": mtime, "config": ProjectConfig(path)} + ProjectConfig._instances[path] = instance + return instance["config"] + + def __repr__(self): + return "" % (self.path or "in-memory") + + def as_tuple(self): + return [(s, self.items(s)) for s in self.sections()] + + def to_json(self): + return json.dumps(self.as_tuple()) + + def update(self, data, clear=False): + assert isinstance(data, list) + if clear: + self._parser = configparser.ConfigParser() + for section, options in data: + if not self._parser.has_section(section): + self._parser.add_section(section) + for option, value in options: + self.set(section, option, value) + + def save(self, path=None): + path = path or self.path + if path in self._instances: + del self._instances[path] + with open(path or self.path, mode="w+", encoding="utf8") as fp: + fp.write(CONFIG_HEADER.strip() + "\n\n") + self._parser.write(fp) + fp.seek(0) + contents = fp.read() + fp.seek(0) + fp.truncate() + fp.write(contents.strip() + "\n") + return True diff --git a/platformio/project/exception.py b/platformio/project/exception.py new file mode 100644 index 0000000..3821c86 --- /dev/null +++ b/platformio/project/exception.py @@ -0,0 +1,54 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException, UserSideException + + +class ProjectError(PlatformioException): + pass + + +class NotPlatformIOProjectError(ProjectError, UserSideException): + MESSAGE = ( + "Not a PlatformIO project. `platformio.ini` file has not been " + "found in current working directory ({0}). To initialize new project " + "please use `platformio project init` command" + ) + + +class InvalidProjectConfError(ProjectError, UserSideException): + MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" + + +class UndefinedEnvPlatformError(ProjectError, UserSideException): + MESSAGE = "Please specify platform for '{0}' environment" + + +class ProjectEnvsNotAvailableError(ProjectError, UserSideException): + MESSAGE = "Please setup environments in `platformio.ini` file" + + +class UnknownEnvNamesError(ProjectError, UserSideException): + MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" + + +class InvalidEnvNameError(ProjectError, UserSideException): + MESSAGE = ( + "Invalid environment name '{0}'. The name can contain " + "alphanumeric, underscore, and hyphen characters (a-z, 0-9, -, _)" + ) + + +class ProjectOptionValueError(ProjectError, UserSideException): + MESSAGE = "{0} for option `{1}`{2}in section [{3}]" diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py new file mode 100644 index 0000000..cb004d2 --- /dev/null +++ b/platformio/project/helpers.py @@ -0,0 +1,196 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import subprocess +from hashlib import sha1 + +from click.testing import CliRunner + +from platformio import __version__, exception, fs +from platformio.compat import IS_MACOS, IS_WINDOWS, hashlib_encode_data +from platformio.project.config import ProjectConfig + + +def get_project_dir(): + return os.getcwd() + + +def is_platformio_project(project_dir=None): + if not project_dir: + project_dir = get_project_dir() + return os.path.isfile(os.path.join(project_dir, "platformio.ini")) + + +def find_project_dir_above(path): + if os.path.isfile(path): + path = os.path.dirname(path) + if is_platformio_project(path): + return path + if os.path.isdir(os.path.dirname(path)): + return find_project_dir_above(os.path.dirname(path)) + return None + + +def get_project_watch_lib_dirs(): + """Used by platformio-node-helpers.project.observer.fetchLibDirs""" + config = ProjectConfig.get_instance() + result = [ + config.get("platformio", "globallib_dir"), + config.get("platformio", "lib_dir"), + ] + libdeps_dir = config.get("platformio", "libdeps_dir") + if not os.path.isdir(libdeps_dir): + return result + for d in os.listdir(libdeps_dir): + if os.path.isdir(os.path.join(libdeps_dir, d)): + result.append(os.path.join(libdeps_dir, d)) + return result + + +get_project_all_lib_dirs = get_project_watch_lib_dirs + + +def get_project_cache_dir(): + """Deprecated, use ProjectConfig.get("platformio", "cache_dir") instead""" + return ProjectConfig.get_instance().get("platformio", "cache_dir") + + +def get_default_projects_dir(): + docs_dir = os.path.join(fs.expanduser("~"), "Documents") + try: + assert IS_WINDOWS + import ctypes.wintypes # pylint: disable=import-outside-toplevel + + buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) + docs_dir = buf.value + except: # pylint: disable=bare-except + if not IS_MACOS: + try: + docs_dir = ( + subprocess.check_output(["xdg-user-dir", "DOCUMENTS"]) + .decode("utf-8") + .strip() + ) + except FileNotFoundError: # command not found + pass + return os.path.join(docs_dir, "PlatformIO", "Projects") + + +def compute_project_checksum(config): + # rebuild when PIO Core version changes + checksum = sha1(hashlib_encode_data(__version__)) + + # configuration file state + config_data = config.to_json() + if IS_WINDOWS: + # issue #4600: fix drive letter + config_data = re.sub( + r"([A-Z]):\\", + lambda match: "%s:\\" % match.group(1).lower(), + config_data, + flags=re.I, + ) + checksum.update(hashlib_encode_data(config_data)) + + # project file structure + check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") + for d in ( + config.get("platformio", "include_dir"), + config.get("platformio", "src_dir"), + config.get("platformio", "lib_dir"), + ): + if not os.path.isdir(d): + continue + chunks = [] + for root, _, files in os.walk(d): + for f in files: + path = os.path.join(root, f) + if path.endswith(check_suffixes): + chunks.append(path) + if not chunks: + continue + chunks_to_str = ",".join(sorted(chunks)) + if IS_WINDOWS: # case insensitive OS + chunks_to_str = chunks_to_str.lower() + checksum.update(hashlib_encode_data(chunks_to_str)) + + return checksum.hexdigest() + + +def load_build_metadata(project_dir, env_or_envs, cache=False, debug=False): + assert env_or_envs + env_names = env_or_envs + if not isinstance(env_names, list): + env_names = [env_names] + + with fs.cd(project_dir): + result = _get_cached_build_metadata(project_dir, env_names) if cache else {} + # incompatible build-type data + for name in list(result.keys()): + build_type = result[name].get("build_type", "") + outdated_conds = [ + not build_type, + debug and "debug" not in build_type, + not debug and "debug" in build_type, + ] + if any(outdated_conds): + del result[name] + missed_env_names = set(env_names) - set(result.keys()) + if missed_env_names: + result.update(_load_build_metadata(project_dir, missed_env_names, debug)) + + if not isinstance(env_or_envs, list) and env_or_envs in result: + return result[env_or_envs] + return result or None + + +# Backward compatibiility with dev-platforms +load_project_ide_data = load_build_metadata + + +def _load_build_metadata(project_dir, env_names, debug=False): + # pylint: disable=import-outside-toplevel + from platformio import app + from platformio.run.cli import cli as cmd_run + + args = ["--project-dir", project_dir, "--target", "__idedata"] + if debug: + args.extend(["--target", "__debug"]) + for name in env_names: + args.extend(["-e", name]) + app.set_session_var("pause_telemetry", True) + result = CliRunner().invoke(cmd_run, args) + app.set_session_var("pause_telemetry", False) + if result.exit_code != 0 and not isinstance( + result.exception, exception.ReturnErrorCode + ): + raise result.exception + if '"includes":' not in result.output: + raise exception.UserSideException(result.output) + return _get_cached_build_metadata(project_dir, env_names) + + +def _get_cached_build_metadata(project_dir, env_names): + build_dir = ProjectConfig.get_instance( + os.path.join(project_dir, "platformio.ini") + ).get("platformio", "build_dir") + result = {} + for name in env_names: + if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")): + continue + result[name] = fs.load_json(os.path.join(build_dir, name, "idedata.json")) + return result diff --git a/platformio/project/integration/__init__.py b/platformio/project/integration/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/project/integration/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py new file mode 100644 index 0000000..d55f9aa --- /dev/null +++ b/platformio/project/integration/generator.py @@ -0,0 +1,176 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import codecs +import os +import sys + +import bottle + +from platformio import fs, util +from platformio.debug.helpers import get_default_debug_env +from platformio.proc import where_is_program +from platformio.project.helpers import load_build_metadata + + +class ProjectGenerator: + def __init__(self, config, env_name, ide, boards=None): + self.config = config + self.project_dir = os.path.dirname(config.path) + self.forced_env_name = env_name + self.env_name = str(env_name or self.get_best_envname(boards)) + self.ide = str(ide) + + def get_best_envname(self, boards=None): + envname = None + default_envs = self.config.default_envs() + if default_envs: + envname = default_envs[0] + if not boards: + return envname + + for env in self.config.envs(): + if not boards: + return env + if not envname: + envname = env + items = self.config.items(env=env, as_dict=True) + if "board" in items and items.get("board") in boards: + return env + return envname + + @staticmethod + def get_ide_tpls_dir(): + return os.path.join(os.path.dirname(__file__), "tpls") + + @classmethod + def get_supported_ides(cls): + tpls_dir = cls.get_ide_tpls_dir() + return sorted( + [ + name + for name in os.listdir(tpls_dir) + if os.path.isdir(os.path.join(tpls_dir, name)) + ] + ) + + @staticmethod + def filter_includes(includes_map, ignore_scopes=None, to_unix_path=True): + ignore_scopes = ignore_scopes or [] + result = [] + for scope, includes in includes_map.items(): + if scope in ignore_scopes: + continue + for include in includes: + if to_unix_path: + include = fs.to_unix_path(include) + if include not in result: + result.append(include) + return result + + def _load_tplvars(self): + tpl_vars = { + "config": self.config, + "systype": util.get_systype(), + "project_name": self.config.get( + "platformio", "name", os.path.basename(self.project_dir) + ), + "project_dir": self.project_dir, + "forced_env_name": self.forced_env_name, + "default_debug_env_name": get_default_debug_env(self.config), + "env_name": self.env_name, + "user_home_dir": os.path.abspath(fs.expanduser("~")), + "platformio_path": sys.argv[0] + if os.path.isfile(sys.argv[0]) + else where_is_program("platformio"), + "env_path": os.getenv("PATH"), + "env_pathsep": os.pathsep, + } + + # default env configuration + tpl_vars.update(self.config.items(env=self.env_name, as_dict=True)) + # build data + tpl_vars.update(load_build_metadata(self.project_dir, self.env_name) or {}) + + with fs.cd(self.project_dir): + tpl_vars.update( + { + "src_files": self.get_src_files(), + "project_src_dir": self.config.get("platformio", "src_dir"), + "project_lib_dir": self.config.get("platformio", "lib_dir"), + "project_test_dir": self.config.get("platformio", "test_dir"), + "project_libdeps_dir": os.path.join( + self.config.get("platformio", "libdeps_dir"), self.env_name + ), + } + ) + + for key, value in tpl_vars.items(): + if key.endswith(("_path", "_dir")): + tpl_vars[key] = fs.to_unix_path(value) + for key in ("src_files", "libsource_dirs"): + if key not in tpl_vars: + continue + tpl_vars[key] = [fs.to_unix_path(inc) for inc in tpl_vars[key]] + + tpl_vars["to_unix_path"] = fs.to_unix_path + tpl_vars["filter_includes"] = self.filter_includes + return tpl_vars + + def get_src_files(self): + result = [] + with fs.cd(self.project_dir): + for root, _, files in os.walk(self.config.get("platformio", "src_dir")): + for f in files: + result.append( + os.path.relpath(os.path.join(os.path.abspath(root), f)) + ) + return result + + def get_tpls(self): + tpls = [] + ide_tpls_dir = os.path.join(self.get_ide_tpls_dir(), self.ide) + for root, _, files in os.walk(ide_tpls_dir): + for f in files: + if not f.endswith(".tpl"): + continue + _relpath = root.replace(ide_tpls_dir, "") + if _relpath.startswith(os.sep): + _relpath = _relpath[1:] + tpls.append((_relpath, os.path.join(root, f))) + return tpls + + def generate(self): + tpl_vars = self._load_tplvars() + for tpl_relpath, tpl_path in self.get_tpls(): + dst_dir = self.project_dir + if tpl_relpath: + dst_dir = os.path.join(self.project_dir, tpl_relpath) + if not os.path.isdir(dst_dir): + os.makedirs(dst_dir) + file_name = os.path.basename(tpl_path)[:-4] + contents = self._render_tpl(tpl_path, tpl_vars) + self._merge_contents(os.path.join(dst_dir, file_name), contents) + + @staticmethod + def _render_tpl(tpl_path, tpl_vars): + with codecs.open(tpl_path, "r", encoding="utf8") as fp: + return bottle.template(fp.read(), **tpl_vars) + + @staticmethod + def _merge_contents(dst_path, contents): + if os.path.basename(dst_path) == ".gitignore" and os.path.isfile(dst_path): + return + with codecs.open(dst_path, "w", encoding="utf8") as fp: + fp.write(contents) diff --git a/platformio/project/integration/tpls/clion/.gitignore.tpl b/platformio/project/integration/tpls/clion/.gitignore.tpl new file mode 100644 index 0000000..03f4a3c --- /dev/null +++ b/platformio/project/integration/tpls/clion/.gitignore.tpl @@ -0,0 +1 @@ +.pio diff --git a/platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl b/platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl new file mode 100644 index 0000000..fcd05d4 --- /dev/null +++ b/platformio/project/integration/tpls/codeblocks/platformio.cbp.tpl @@ -0,0 +1,64 @@ + + + + + + diff --git a/platformio/project/integration/tpls/eclipse/.cproject.tpl b/platformio/project/integration/tpls/eclipse/.cproject.tpl new file mode 100644 index 0000000..d11e95c --- /dev/null +++ b/platformio/project/integration/tpls/eclipse/.cproject.tpl @@ -0,0 +1,316 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + platformio + -f -c eclipse + run --target program + true + false + false + + + platformio + -f -c eclipse + run --target uploadfs + true + false + false + + + platformio + -f -c eclipse + run + true + false + false + + + platformio + -f -c eclipse + run --verbose + true + false + false + + + platformio + -f -c eclipse + run --target upload + true + false + false + + + platformio + -f -c eclipse + run --target upload --verbose + true + false + false + + + platformio + -f -c eclipse + run --target clean + true + false + false + + + platformio + -f -c eclipse + test + true + false + false + + + platformio + -f -c eclipse + remote run --target upload + true + false + false + + + platformio + -f -c eclipse + init --ide eclipse + true + false + false + + + platformio + -f -c eclipse + device list + true + false + false + + + platformio + -f -c eclipse + lib update + true + false + false + + + platformio + -f -c eclipse + update + true + false + false + + + platformio + -f -c eclipse + upgrade + true + false + false + + + + diff --git a/platformio/project/integration/tpls/eclipse/.project.tpl b/platformio/project/integration/tpls/eclipse/.project.tpl new file mode 100644 index 0000000..ac49f54 --- /dev/null +++ b/platformio/project/integration/tpls/eclipse/.project.tpl @@ -0,0 +1,27 @@ + + + {{project_name}} + + + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + diff --git a/platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl b/platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl new file mode 100644 index 0000000..6b41950 --- /dev/null +++ b/platformio/project/integration/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl new file mode 100644 index 0000000..f2123be --- /dev/null +++ b/platformio/project/integration/tpls/eclipse/.settings/language.settings.xml.tpl @@ -0,0 +1,36 @@ +% cxx_stds = [arg for arg in cxx_flags if arg.startswith("-std=")] +% cxx_std = cxx_stds[-1] if cxx_stds else "" +% +% if cxx_path.startswith(user_home_dir): +% if "windows" in systype: +% cxx_path = "${USERPROFILE}" + cxx_path.replace(user_home_dir, "") +% else: +% cxx_path = "${HOME}" + cxx_path.replace(user_home_dir, "") +% end +% end +% + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl new file mode 100644 index 0000000..13b8c38 --- /dev/null +++ b/platformio/project/integration/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl @@ -0,0 +1,11 @@ +eclipse.preferences.version=1 +environment/project/0.910961921/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} +environment/project/0.910961921/PATH/operation=replace +environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH} +environment/project/0.910961921/append=true +environment/project/0.910961921/appendContributed=true +environment/project/0.910961921.1363900502/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} +environment/project/0.910961921.1363900502/PATH/operation=replace +environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH} +environment/project/0.910961921.1363900502/append=true +environment/project/0.910961921.1363900502/appendContributed=true \ No newline at end of file diff --git a/platformio/project/integration/tpls/emacs/.ccls.tpl b/platformio/project/integration/tpls/emacs/.ccls.tpl new file mode 100644 index 0000000..94d3768 --- /dev/null +++ b/platformio/project/integration/tpls/emacs/.ccls.tpl @@ -0,0 +1,14 @@ +% from platformio.compat import shlex_join +% +clang + +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} + +% for include in filter_includes(includes): +-I{{ !include }} +% end + +% for define in defines: +-D{{ !define }} +% end diff --git a/platformio/project/integration/tpls/emacs/.gitignore.tpl b/platformio/project/integration/tpls/emacs/.gitignore.tpl new file mode 100644 index 0000000..6f8bafd --- /dev/null +++ b/platformio/project/integration/tpls/emacs/.gitignore.tpl @@ -0,0 +1,3 @@ +.pio +.clang_complete +.ccls diff --git a/platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl new file mode 100644 index 0000000..099e0e4 --- /dev/null +++ b/platformio/project/integration/tpls/netbeans/nbproject/configurations.xml.tpl @@ -0,0 +1,73 @@ + + + + + platformio.ini + + + nbproject/private/launcher.properties + + + ^(nbproject|.pio)$ + + . + + + + + + default + false + false + + + true + true + + + + . + "{{platformio_path}}" -f -c netbeans run + "{{platformio_path}}" -f -c netbeans run --target clean + + + % cleaned_includes = filter_includes(includes) + + src + % for include in cleaned_includes: + {{include}} + % end + + + % for define in defines: + {{define}} + % end + + + + + src + % for include in cleaned_includes: + {{include}} + % end + + + % for define in defines: + {{define}} + % end + + + + + . + + + + + + + + diff --git a/platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl new file mode 100644 index 0000000..2beef95 --- /dev/null +++ b/platformio/project/integration/tpls/netbeans/nbproject/private/configurations.xml.tpl @@ -0,0 +1,61 @@ + + + + + + + + + platformio.ini + + + + + + + localhost + {{4 if "darwin" in systype else 2 if "linux" in systype else 3}} + + + + . + ${AUTO_FOLDER} + + ${AUTO_FOLDER} + + ${MAKE} ${ITEM_NAME}.o + ${AUTO_COMPILE} + + ${AUTO_COMPILE} + + + + + + + + + + + + + + + gdb + + + + "${OUTPUT_PATH}" + {{platformio_path}} -f -c netbeans run --target upload + + {{platformio_path}} -f -c netbeans run --target upload + . + false + 0 + 0 + + + + + + diff --git a/platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl new file mode 100644 index 0000000..6cc2127 --- /dev/null +++ b/platformio/project/integration/tpls/netbeans/nbproject/private/launcher.properties.tpl @@ -0,0 +1,40 @@ +# Launchers File syntax: +# +# [Must-have property line] +# launcher1.runCommand= +# [Optional extra properties] +# launcher1.displayName= +# launcher1.buildCommand= +# launcher1.runDir= +# launcher1.symbolFiles= +# launcher1.env.= +# (If this value is quoted with ` it is handled as a native command which execution result will become the value) +# [Common launcher properties] +# common.runDir= +# (This value is overwritten by a launcher specific runDir value if the latter exists) +# common.env.= +# (Environment variables from common launcher are merged with launcher specific variables) +# common.symbolFiles= +# (This value is overwritten by a launcher specific symbolFiles value if the latter exists) +# +# In runDir, symbolFiles and env fields you can use these macroses: +# ${PROJECT_DIR} - project directory absolute path +# ${OUTPUT_PATH} - linker output path (relative to project directory path) +# ${OUTPUT_BASENAME}- linker output filename +# ${TESTDIR} - test files directory (relative to project directory path) +# ${OBJECTDIR} - object files directory (relative to project directory path) +# ${CND_DISTDIR} - distribution directory (relative to project directory path) +# ${CND_BUILDDIR} - build directory (relative to project directory path) +# ${CND_PLATFORM} - platform name +# ${CND_CONF} - configuration name +# ${CND_DLIB_EXT} - dynamic library extension +# +# All the project launchers must be listed in the file! +# +# launcher1.runCommand=... +# launcher2.runCommand=... +# ... +# common.runDir=... +# common.env.KEY=VALUE + +# launcher1.runCommand= \ No newline at end of file diff --git a/platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl new file mode 100644 index 0000000..4f3fdc6 --- /dev/null +++ b/platformio/project/integration/tpls/netbeans/nbproject/private/private.xml.tpl @@ -0,0 +1,10 @@ + + + + true + + + 0 + 0 + + diff --git a/platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl b/platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl new file mode 100644 index 0000000..ad4ea18 --- /dev/null +++ b/platformio/project/integration/tpls/netbeans/nbproject/project.xml.tpl @@ -0,0 +1,26 @@ + + + org.netbeans.modules.cnd.makeproject + + + {{project_name}} + + + + UTF-8 + + + . + + + + Default + 0 + + + + false + + + + diff --git a/platformio/project/integration/tpls/qtcreator/.gitignore.tpl b/platformio/project/integration/tpls/qtcreator/.gitignore.tpl new file mode 100644 index 0000000..0c1fe4a --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/.gitignore.tpl @@ -0,0 +1,2 @@ +.pio +.qtc_clangd diff --git a/platformio/project/integration/tpls/qtcreator/Makefile.tpl b/platformio/project/integration/tpls/qtcreator/Makefile.tpl new file mode 100644 index 0000000..d6d6344 --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/Makefile.tpl @@ -0,0 +1,12 @@ +all: + platformio -c qtcreator run + +# regenerate project files to reflect platformio.ini changes +project-update: + @echo "This will overwrite project metadata files. Are you sure? [y/N] " \ + && read ans && [ $${ans:-'N'} = 'y' ] + platformio project init --ide qtcreator + +# forward any other target (clean, build, etc.) to pio run +{{'%'}}: + platformio -c qtcreator run --target $* diff --git a/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl new file mode 100644 index 0000000..024c813 --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/platformio.cflags.tpl @@ -0,0 +1,3 @@ +% from platformio.compat import shlex_join +% +{{shlex_join(cc_flags).replace('-mlongcalls', '-mlong-calls')}} diff --git a/platformio/project/integration/tpls/qtcreator/platformio.config.tpl b/platformio/project/integration/tpls/qtcreator/platformio.config.tpl new file mode 100644 index 0000000..936fdad --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/platformio.config.tpl @@ -0,0 +1,8 @@ +% for define in defines: +% tokens = define.split("=", 1) +% if len(tokens) > 1: +#define {{tokens[0].strip()}} {{!tokens[1].strip()}} +% else: +#define {{define}} +% end +% end diff --git a/platformio/project/integration/tpls/qtcreator/platformio.creator.tpl b/platformio/project/integration/tpls/qtcreator/platformio.creator.tpl new file mode 100644 index 0000000..d0cc464 --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/platformio.creator.tpl @@ -0,0 +1,2 @@ +[General] + diff --git a/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl new file mode 100644 index 0000000..551f02e --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/platformio.cxxflags.tpl @@ -0,0 +1,3 @@ +% from platformio.compat import shlex_join +% +{{shlex_join(cxx_flags).replace('-mlongcalls', '-mlong-calls')}} diff --git a/platformio/project/integration/tpls/qtcreator/platformio.files.tpl b/platformio/project/integration/tpls/qtcreator/platformio.files.tpl new file mode 100644 index 0000000..b5c0265 --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/platformio.files.tpl @@ -0,0 +1,6 @@ +Makefile +platformio.ini +.gitignore +% for file in src_files: +{{file}} +% end diff --git a/platformio/project/integration/tpls/qtcreator/platformio.includes.tpl b/platformio/project/integration/tpls/qtcreator/platformio.includes.tpl new file mode 100644 index 0000000..435cab1 --- /dev/null +++ b/platformio/project/integration/tpls/qtcreator/platformio.includes.tpl @@ -0,0 +1,4 @@ +./ +% for include in filter_includes(includes): +{{include}} +% end diff --git a/platformio/project/integration/tpls/sublimetext/.ccls.tpl b/platformio/project/integration/tpls/sublimetext/.ccls.tpl new file mode 100644 index 0000000..94d3768 --- /dev/null +++ b/platformio/project/integration/tpls/sublimetext/.ccls.tpl @@ -0,0 +1,14 @@ +% from platformio.compat import shlex_join +% +clang + +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} + +% for include in filter_includes(includes): +-I{{ !include }} +% end + +% for define in defines: +-D{{ !define }} +% end diff --git a/platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl b/platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl new file mode 100644 index 0000000..2b5d61d --- /dev/null +++ b/platformio/project/integration/tpls/sublimetext/platformio.sublime-project.tpl @@ -0,0 +1,108 @@ +{ + "build_systems": + [ + { + "cmd": + [ + "{{ platformio_path }}", + "-c", "sublimetext", + "run" + ], + "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", + "name": "PlatformIO", + "variants": + [ + { + "cmd": + [ + "{{ platformio_path }}", + "-c", "sublimetext", + "run" + ], + "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", + "name": "Build" + }, + { + "cmd": + [ + "{{ platformio_path }}", + "-c", "sublimetext", + "run", + "--target", + "upload" + ], + "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", + "name": "Upload" + }, + { + "cmd": + [ + "{{ platformio_path }}", + "-c", "sublimetext", + "run", + "--target", + "clean" + ], + "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", + "name": "Clean" + }, + { + "cmd": + [ + "{{ platformio_path }}", + "-c", "sublimetext", + "test" + ], + "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", + "name": "Test" + }, + { + "cmd": + [ + "{{ platformio_path }}", + "-c", "sublimetext", + "run", + "--target", + "uploadfs" + ], + "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", + "name": "Upload SPIFFS image" + }, + { + "cmd": + [ + "{{ platformio_path }}", + "-c", "sublimetext", + "update" + ], + "file_regex": "^(..[^:\n]*):([0-9]+):?([0-9]+)?:? (.*)$", + "name": "Update platforms and libraries" + }, + { + "cmd": + [ + "{{ platformio_path }}", + "-c", "sublimetext", + "upgrade" + ], + "name": "Upgrade PlatformIO Core" + } + ], + "working_dir": "${project_path:${folder}}", + "selector": "source.c, source.c++" + } + ], + "folders": + [ + { + "path": "." + } + ], + "settings": + { + "sublimegdb_workingdir": "{{project_dir}}", + "sublimegdb_exec_cmd": "", + "sublimegdb_commandline": "{{ platformio_path }} -f -c sublimetext debug --interface=gdb --interpreter=mi -x .pioinit" + + } +} diff --git a/platformio/project/integration/tpls/vim/.ccls.tpl b/platformio/project/integration/tpls/vim/.ccls.tpl new file mode 100644 index 0000000..94d3768 --- /dev/null +++ b/platformio/project/integration/tpls/vim/.ccls.tpl @@ -0,0 +1,14 @@ +% from platformio.compat import shlex_join +% +clang + +{{"%c"}} {{ shlex_join(cc_flags) }} +{{"%cpp"}} {{ shlex_join(cxx_flags) }} + +% for include in filter_includes(includes): +-I{{ !include }} +% end + +% for define in defines: +-D{{ !define }} +% end diff --git a/platformio/project/integration/tpls/vim/.gitignore.tpl b/platformio/project/integration/tpls/vim/.gitignore.tpl new file mode 100644 index 0000000..1159b2d --- /dev/null +++ b/platformio/project/integration/tpls/vim/.gitignore.tpl @@ -0,0 +1,4 @@ +.pio +.clang_complete +.gcc-flags.json +.ccls diff --git a/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl new file mode 100644 index 0000000..cce99af --- /dev/null +++ b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.filters.tpl @@ -0,0 +1,29 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx;ino;pde + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + + + + % for file in src_files: + + % if any(file.endswith(".%s" % e) for e in ("h", "hh", "hpp", "inc")): + + Header Files + + % else: + + Source Files + + %end + + % end + diff --git a/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl new file mode 100644 index 0000000..828192c --- /dev/null +++ b/platformio/project/integration/tpls/visualstudio/platformio.vcxproj.tpl @@ -0,0 +1,75 @@ + + + + {{env_path}} + + + + Debug + Win32 + + + Release + Win32 + + + + {0FA9C3A8-452B-41EF-A418-9102B170F49F} + MakeFileProj + + + + Makefile + true + v120 + + + Makefile + false + v120 + + + + + + + + + + + + + platformio -f -c visualstudio run + platformio -f -c visualstudio run --target clean + {{!";".join(defines)}} + % cleaned_includes = filter_includes(includes) + {{";".join(["$(HOMEDRIVE)$(HOMEPATH)%s" % i.replace(user_home_dir, "") if i.startswith(user_home_dir) else i for i in cleaned_includes])}} + + + platformio run + platformio run --target clean + {{!";".join(defines)}} + {{";".join(["$(HOMEDRIVE)$(HOMEPATH)%s" % i.replace(user_home_dir, "") if i.startswith(user_home_dir) else i for i in cleaned_includes])}} + + + + + + + + + + % for file in src_files: + + % if any(file.endswith(".%s" % e) for e in ("h", "hh", "hpp", "inc")): + + Header Files + + % else: + + Source Files + + %end + + % end + diff --git a/platformio/project/integration/tpls/vscode/.gitignore.tpl b/platformio/project/integration/tpls/vscode/.gitignore.tpl new file mode 100644 index 0000000..89cc49c --- /dev/null +++ b/platformio/project/integration/tpls/vscode/.gitignore.tpl @@ -0,0 +1,5 @@ +.pio +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode/ipch diff --git a/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl new file mode 100644 index 0000000..0be8a3b --- /dev/null +++ b/platformio/project/integration/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -0,0 +1,138 @@ +% import os +% import platform +% +% systype = platform.system().lower() +% +% cpp_standards_remap = { +% "c++0x": "c++11", +% "c++1y": "c++14", +% "c++1z": "c++17", +% "c++2a": "c++20", +% "c++2b": "c++23", +% "gnu++0x": "gnu++11", +% "gnu++1y": "gnu++14", +% "gnu++1z": "gnu++17", +% "gnu++2a": "gnu++20", +% "gnu++2b": "gnu++23" +% } +% +% def _escape(text): +% return to_unix_path(text).replace('"', '\\"') +% end +% +% def filter_args(args, allowed, ignore=None): +% if not allowed: +% return [] +% end +% +% ignore = ignore or [] +% result = [] +% i = 0 +% length = len(args) +% while(i < length): +% if any(args[i].startswith(f) for f in allowed) and not any( +% args[i].startswith(f) for f in ignore): +% result.append(args[i]) +% if i + 1 < length and not args[i + 1].startswith("-"): +% i += 1 +% result.append(args[i]) +% end +% end +% i += 1 +% end +% return result +% end +% +% def _find_abs_path(inc, inc_paths): +% for path in inc_paths: +% if os.path.isfile(os.path.join(path, inc)): +% return os.path.join(path, inc) +% end +% end +% return inc +% end +% +% def _find_forced_includes(flags, inc_paths): +% result = [] +% include_args = ("-include", "-imacros") +% for f in filter_args(flags, include_args): +% for arg in include_args: +% inc = "" +% if f.startswith(arg) and f.split(arg)[1].strip(): +% inc = f.split(arg)[1].strip() +% elif not f.startswith("-"): +% inc = f +% end +% if inc: +% result.append(_find_abs_path(inc, inc_paths)) +% break +% end +% end +% end +% return result +% end +% +% cleaned_includes = filter_includes(includes, ["toolchain"]) +% +% cc_stds = [arg[5:] for arg in cc_flags if arg.startswith("-std=")] +% cxx_stds = [arg[5:] for arg in cxx_flags if arg.startswith("-std=")] +% forced_includes = _find_forced_includes( +% filter_args(cc_flags, ["-include", "-imacros"]), cleaned_includes) +% +// +// !!! WARNING !!! AUTO-GENERATED FILE! +// PLEASE DO NOT MODIFY IT AND USE "platformio.ini": +// https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags +// +{ + "configurations": [ + { + "name": "PlatformIO", + "includePath": [ +% for include in cleaned_includes: + "{{ include }}", +% end + "" + ], + "browse": { + "limitSymbolsToIncludedHeaders": true, + "path": [ +% for include in cleaned_includes: + "{{ include }}", +% end + "" + ] + }, + "defines": [ +% for define in defines: + "{{! _escape(define) }}", +% end + "" + ], +% if cc_stds: + "cStandard": "{{ cc_stds[-1] }}", +% end +% if cxx_stds: + "cppStandard": "{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}", +% end +% if forced_includes: + "forcedInclude": [ +% for include in forced_includes: + "{{ include }}", +% end + "" + ], +% end + "compilerPath": "{{ cc_path }}", + "compilerArgs": [ +% for flag in [ +% f for f in filter_args(cc_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) +% ]: + "{{ flag }}", +% end + "" + ] + } + ], + "version": 4 +} diff --git a/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl new file mode 100644 index 0000000..8d8e8a0 --- /dev/null +++ b/platformio/project/integration/tpls/vscode/.vscode/extensions.json.tpl @@ -0,0 +1,35 @@ +% import json +% import os +% import re +% +% recommendations = set(["platformio.platformio-ide"]) +% unwantedRecommendations = set(["ms-vscode.cpptools-extension-pack"]) +% previous_json = os.path.join(project_dir, ".vscode", "extensions.json") +% if os.path.isfile(previous_json): +% fp = open(previous_json) +% contents = re.sub(r"^\s*//.*$", "", fp.read(), flags=re.M).strip() +% fp.close() +% if contents: +% try: +% data = json.loads(contents) +% recommendations |= set(data.get("recommendations", [])) +% unwantedRecommendations |= set(data.get("unwantedRecommendations", [])) +% except ValueError: +% pass +% end +% end +% end +{ + // See http://go.microsoft.com/fwlink/?LinkId=827846 + // for the documentation about the extensions.json format + "recommendations": [ +% for i, item in enumerate(sorted(recommendations)): + "{{ item }}"{{ ("," if (i + 1) < len(recommendations) else "") }} +% end + ], + "unwantedRecommendations": [ +% for i, item in enumerate(sorted(unwantedRecommendations)): + "{{ item }}"{{ ("," if (i + 1) < len(unwantedRecommendations) else "") }} +% end + ] +} diff --git a/platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl b/platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl new file mode 100644 index 0000000..4bd188d --- /dev/null +++ b/platformio/project/integration/tpls/vscode/.vscode/launch.json.tpl @@ -0,0 +1,99 @@ +% import codecs +% import json +% import os +% +% def _escape(text): +% return text.replace('"', '\"') +% end +% +% def _escape_path(path): +% return path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"') +% end +% +% def get_pio_configurations(): +% predebug = { +% "type": "platformio-debug", +% "request": "launch", +% "name": "PIO Debug (skip Pre-Debug)", +% "executable": _escape_path(prog_path), +% "projectEnvName": env_name if forced_env_name else default_debug_env_name, +% "toolchainBinDir": _escape_path(os.path.dirname(cc_path)), +% "internalConsoleOptions": "openOnSessionStart", +% } +% +% if svd_path: +% predebug["svdPath"] = _escape_path(svd_path) +% end +% debug = predebug.copy() +% debug["name"] = "PIO Debug" +% debug["preLaunchTask"] = { +% "type": "PlatformIO", +% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and forced_env_name else "Pre-Debug", +% } +% noloading = predebug.copy() +% noloading["name"] = "PIO Debug (without uploading)" +% noloading["loadMode"] = "manual" +% return [debug, predebug, noloading] +% end +% +% def _remove_comments(lines): +% data = "" +% for line in lines: +% line = line.strip() +% if not line.startswith("//"): +% data += line +% end +% end +% return data +% end +% +% def _contains_external_configurations(launch_config): +% return any( +% c.get("type", "") != "platformio-debug" +% for c in launch_config.get("configurations", []) +% ) +% end +% +% def _remove_pio_configurations(launch_config): +% if "configurations" not in launch_config: +% return launch_config +% end +% +% external_configurations = [ +% config +% for config in launch_config["configurations"] +% if config.get("type", "") != "platformio-debug" +% ] +% +% launch_config["configurations"] = external_configurations +% return launch_config +% end +% +% def get_launch_configuration(): +% launch_config = {"version": "0.2.0", "configurations": []} +% launch_file = os.path.join(project_dir, ".vscode", "launch.json") +% if os.path.isfile(launch_file): +% with codecs.open(launch_file, "r", encoding="utf8") as fp: +% launch_data = _remove_comments(fp.readlines()) +% try: +% prev_config = json.loads(launch_data) +% if _contains_external_configurations(prev_config): +% launch_config = _remove_pio_configurations(prev_config) +% end +% except: +% pass +% end +% end +% end +% launch_config["configurations"].extend(get_pio_configurations()) +% return launch_config +% end +% +// AUTOMATICALLY GENERATED FILE. PLEASE DO NOT MODIFY IT MANUALLY +// +// PIO Unified Debugger +// +// Documentation: https://docs.platformio.org/page/plus/debugging.html +// Configuration: https://docs.platformio.org/page/projectconf/section_env_debug.html + +{{ json.dumps(get_launch_configuration(), indent=4, ensure_ascii=False) }} diff --git a/platformio/project/options.py b/platformio/project/options.py new file mode 100644 index 0000000..2e13815 --- /dev/null +++ b/platformio/project/options.py @@ -0,0 +1,852 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-builtin, too-many-arguments + +import hashlib +import os +from collections import OrderedDict + +import click + +from platformio import fs +from platformio.compat import IS_WINDOWS, hashlib_encode_data + + +class ConfigOption: # pylint: disable=too-many-instance-attributes + def __init__( + self, + scope, + group, + name, + description, + type=str, + multiple=False, + sysenvvar=None, + buildenvvar=None, + oldnames=None, + default=None, + validate=None, + ): + self.scope = scope + self.group = group + self.name = name + self.description = description + self.type = type + self.multiple = multiple + self.sysenvvar = sysenvvar + self.buildenvvar = buildenvvar + self.oldnames = oldnames + self.default = default + self.validate = validate + + def as_dict(self): + result = dict( + scope=self.scope, + group=self.group, + name=self.name, + description=self.description, + type="string", + multiple=self.multiple, + sysenvvar=self.sysenvvar, + default=self.default() if callable(self.default) else self.default, + ) + if isinstance(self.type, click.ParamType): + result["type"] = self.type.name + if isinstance(self.type, (click.IntRange, click.FloatRange)): + result["min"] = self.type.min + result["max"] = self.type.max + if isinstance(self.type, click.Choice): + result["choices"] = self.type.choices + return result + + +def ConfigPlatformioOption(*args, **kwargs): + return ConfigOption("platformio", *args, **kwargs) + + +def ConfigEnvOption(*args, **kwargs): + return ConfigOption("env", *args, **kwargs) + + +def calculate_path_hash(path): + return "%s-%s" % ( + os.path.basename(path), + hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10], + ) + + +def expand_dir_templates(path): + project_dir = os.getcwd() + tpls = { + "$PROJECT_DIR": lambda: project_dir, + "$PROJECT_HASH": lambda: calculate_path_hash(project_dir), + } + done = False + while not done: + done = True + for tpl, cb in tpls.items(): + if tpl not in path: + continue + path = path.replace(tpl, cb()) + done = False + return path + + +def validate_dir(path): + if not path: + return path + # if not all values expanded, ignore validation + if "${" in path and "}" in path: + return path + if path.startswith("~"): + path = fs.expanduser(path) + if "$" in path: + path = expand_dir_templates(path) + return os.path.abspath(path) + + +def get_default_core_dir(): + path = os.path.join(fs.expanduser("~"), ".platformio") + if IS_WINDOWS: + win_core_dir = os.path.splitdrive(path)[0] + "\\.platformio" + if os.path.isdir(win_core_dir): + return win_core_dir + return path + + +ProjectOptions = OrderedDict( + [ + ("%s.%s" % (option.scope, option.name), option) + for option in [ + # + # [platformio] + # + ConfigPlatformioOption( + group="generic", + name="name", + description="A project name", + ), + ConfigPlatformioOption( + group="generic", + name="description", + description="Describe a project with a short information", + ), + ConfigPlatformioOption( + group="generic", + name="default_envs", + description=( + "Configure a list with environments which PlatformIO should " + "process by default" + ), + oldnames=["env_default"], + multiple=True, + sysenvvar="PLATFORMIO_DEFAULT_ENVS", + ), + ConfigPlatformioOption( + group="generic", + name="extra_configs", + description=( + "Extend main configuration with the extra configuration files" + ), + multiple=True, + ), + # Dirs + ConfigPlatformioOption( + group="directory", + name="core_dir", + description=( + "PlatformIO Core location where it keeps installed development " + "platforms, packages, global libraries, " + "and other internal information" + ), + oldnames=["home_dir"], + sysenvvar="PLATFORMIO_CORE_DIR", + default=get_default_core_dir, + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="globallib_dir", + description=( + "A library folder/storage where PlatformIO Library Dependency " + "Finder (LDF) looks for global libraries" + ), + sysenvvar="PLATFORMIO_GLOBALLIB_DIR", + default=os.path.join("${platformio.core_dir}", "lib"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="platforms_dir", + description=( + "A location where PlatformIO Core keeps installed development " + "platforms" + ), + sysenvvar="PLATFORMIO_PLATFORMS_DIR", + default=os.path.join("${platformio.core_dir}", "platforms"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="packages_dir", + description=( + "A location where PlatformIO Core keeps installed packages" + ), + sysenvvar="PLATFORMIO_PACKAGES_DIR", + default=os.path.join("${platformio.core_dir}", "packages"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="cache_dir", + description=( + "A location where PlatformIO Core stores caching information " + "(requests to PlatformIO Registry, downloaded packages and " + "other service information)" + ), + sysenvvar="PLATFORMIO_CACHE_DIR", + default=os.path.join("${platformio.core_dir}", ".cache"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="build_cache_dir", + description=( + "A location where PlatformIO Core keeps derived files from a " + "build system (objects, firmwares, ELFs) and caches them between " + "build environments" + ), + sysenvvar="PLATFORMIO_BUILD_CACHE_DIR", + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="workspace_dir", + description=( + "A path to a project workspace directory where PlatformIO keeps " + "by default compiled objects, static libraries, firmwares, and " + "external library dependencies" + ), + sysenvvar="PLATFORMIO_WORKSPACE_DIR", + default=os.path.join("$PROJECT_DIR", ".pio"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="build_dir", + description=( + "PlatformIO Build System uses this folder for project environments" + " to store compiled object files, static libraries, firmwares, " + "and other cached information" + ), + sysenvvar="PLATFORMIO_BUILD_DIR", + default=os.path.join("${platformio.workspace_dir}", "build"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="libdeps_dir", + description=( + "Internal storage where Library Manager will install project " + "dependencies declared via `lib_deps` option" + ), + sysenvvar="PLATFORMIO_LIBDEPS_DIR", + default=os.path.join("${platformio.workspace_dir}", "libdeps"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="include_dir", + description=( + "A default location for project header files. PlatformIO Build " + "System automatically adds this path to CPPPATH scope" + ), + sysenvvar="PLATFORMIO_INCLUDE_DIR", + default=os.path.join("$PROJECT_DIR", "include"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="src_dir", + description=( + "A default location where PlatformIO Build System looks for the " + "project C/C++ source files" + ), + sysenvvar="PLATFORMIO_SRC_DIR", + default=os.path.join("$PROJECT_DIR", "src"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="lib_dir", + description="A storage for the custom/private project libraries", + sysenvvar="PLATFORMIO_LIB_DIR", + default=os.path.join("$PROJECT_DIR", "lib"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="data_dir", + description=( + "A data directory to store contents which can be uploaded to " + "file system (SPIFFS, etc.)" + ), + sysenvvar="PLATFORMIO_DATA_DIR", + default=os.path.join("$PROJECT_DIR", "data"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="test_dir", + description=( + "A location where PlatformIO Unit Testing engine looks for " + "test source files" + ), + sysenvvar="PLATFORMIO_TEST_DIR", + default=os.path.join("$PROJECT_DIR", "test"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="boards_dir", + description="A storage for custom board manifests", + sysenvvar="PLATFORMIO_BOARDS_DIR", + default=os.path.join("$PROJECT_DIR", "boards"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="monitor_dir", + description="A storage for custom monitor filters", + sysenvvar="PLATFORMIO_MONITOR_DIR", + default=os.path.join("$PROJECT_DIR", "monitor"), + validate=validate_dir, + ), + ConfigPlatformioOption( + group="directory", + name="shared_dir", + description=( + "A location which PlatformIO Remote Development service uses to " + "synchronize extra files between remote machines" + ), + sysenvvar="PLATFORMIO_SHARED_DIR", + default=os.path.join("$PROJECT_DIR", "shared"), + validate=validate_dir, + ), + # + # [env] + # + # Platform + ConfigEnvOption( + group="platform", + name="platform", + description="A name or specification of development platform", + buildenvvar="PIOPLATFORM", + ), + ConfigEnvOption( + group="platform", + name="platform_packages", + description="Custom packages and specifications", + multiple=True, + ), + # Board + ConfigEnvOption( + group="platform", + name="board", + description="A board ID", + buildenvvar="BOARD", + ), + ConfigEnvOption( + group="platform", + name="framework", + description="A list of project dependent frameworks", + multiple=True, + buildenvvar="PIOFRAMEWORK", + ), + ConfigEnvOption( + group="platform", + name="board_build.mcu", + description="A custom board MCU", + oldnames=["board_mcu"], + buildenvvar="BOARD_MCU", + ), + ConfigEnvOption( + group="platform", + name="board_build.f_cpu", + description="A custom MCU frequency", + oldnames=["board_f_cpu"], + buildenvvar="BOARD_F_CPU", + ), + ConfigEnvOption( + group="platform", + name="board_build.f_flash", + description="A custom flash frequency", + oldnames=["board_f_flash"], + buildenvvar="BOARD_F_FLASH", + ), + ConfigEnvOption( + group="platform", + name="board_build.flash_mode", + description="A custom flash mode", + oldnames=["board_flash_mode"], + buildenvvar="BOARD_FLASH_MODE", + ), + # Build + ConfigEnvOption( + group="build", + name="build_type", + description="Project build configuration", + type=click.Choice(["release", "test", "debug"]), + default="release", + ), + ConfigEnvOption( + group="build", + name="build_flags", + description=( + "Custom build flags/options for preprocessing, compilation, " + "assembly, and linking processes" + ), + multiple=True, + sysenvvar="PLATFORMIO_BUILD_FLAGS", + buildenvvar="BUILD_FLAGS", + ), + ConfigEnvOption( + group="build", + name="build_src_flags", + oldnames=["src_build_flags"], + description=( + "The same as `build_flags` but configures flags the only for " + "project source files in the `src` folder" + ), + multiple=True, + sysenvvar="PLATFORMIO_BUILD_SRC_FLAGS", + buildenvvar="SRC_BUILD_FLAGS", + ), + ConfigEnvOption( + group="build", + name="build_unflags", + description="A list with flags/option which should be removed", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_UNFLAGS", + buildenvvar="BUILD_UNFLAGS", + ), + ConfigEnvOption( + group="build", + name="build_src_filter", + oldnames=["src_filter"], + description=( + "Control which source files from the `src` folder should " + "be included/excluded from a build process" + ), + multiple=True, + sysenvvar="PLATFORMIO_BUILD_SRC_FILTER", + buildenvvar="SRC_FILTER", + default="+<*> -<.git/> -<.svn/>", + ), + ConfigEnvOption( + group="build", + name="targets", + description="A custom list of targets for PlatformIO Build System", + multiple=True, + ), + # Upload + ConfigEnvOption( + group="upload", + name="upload_port", + description=( + "An upload port which `uploader` tool uses for a firmware flashing" + ), + sysenvvar="PLATFORMIO_UPLOAD_PORT", + buildenvvar="UPLOAD_PORT", + ), + ConfigEnvOption( + group="upload", + name="upload_protocol", + description="A protocol that `uploader` tool uses to talk to a board", + buildenvvar="UPLOAD_PROTOCOL", + ), + ConfigEnvOption( + group="upload", + name="upload_speed", + description=( + "A connection speed (baud rate) which `uploader` tool uses when " + "sending firmware to a board" + ), + type=click.INT, + buildenvvar="UPLOAD_SPEED", + ), + ConfigEnvOption( + group="upload", + name="upload_flags", + description="An extra flags for `uploader` tool", + multiple=True, + sysenvvar="PLATFORMIO_UPLOAD_FLAGS", + buildenvvar="UPLOAD_FLAGS", + ), + ConfigEnvOption( + group="upload", + name="upload_resetmethod", + description="A custom reset method", + buildenvvar="UPLOAD_RESETMETHOD", + ), + ConfigEnvOption( + group="upload", + name="upload_command", + description=( + "A custom upload command which overwrites a default from " + "development platform" + ), + buildenvvar="UPLOADCMD", + ), + # Monitor + ConfigEnvOption( + group="monitor", + name="monitor_port", + description="A port, a number or a device name", + ), + ConfigEnvOption( + group="monitor", + name="monitor_speed", + description="A monitor speed (baud rate)", + type=click.INT, + oldnames=["monitor_baud"], + default=9600, + ), + ConfigEnvOption( + group="monitor", + name="monitor_parity", + description="A monitor parity checking", + type=click.Choice(["N", "E", "O", "S", "M"]), + default="N", + ), + ConfigEnvOption( + group="monitor", + name="monitor_filters", + description=( + "Apply the filters and text transformations to monitor output" + ), + multiple=True, + ), + ConfigEnvOption( + group="monitor", + name="monitor_rts", + description="A monitor initial RTS line state", + type=click.IntRange(0, 1), + ), + ConfigEnvOption( + group="monitor", + name="monitor_dtr", + description="A monitor initial DTR line state", + type=click.IntRange(0, 1), + ), + ConfigEnvOption( + group="monitor", + name="monitor_eol", + description="A monitor end of line mode", + type=click.Choice(["CR", "LF", "CRLF"]), + default="CRLF", + ), + ConfigEnvOption( + group="monitor", + name="monitor_raw", + description="Disable encodings/transformations of device output", + type=click.BOOL, + default=False, + ), + ConfigEnvOption( + group="monitor", + name="monitor_echo", + description="Enable a monitor local echo", + type=click.BOOL, + default=False, + ), + ConfigEnvOption( + group="monitor", + name="monitor_encoding", + description="Custom encoding (e.g. hexlify, Latin1, UTF-8)", + default="UTF-8", + ), + # Library + ConfigEnvOption( + group="library", + name="lib_deps", + description=( + "A list of project library dependencies which should be installed " + "automatically before a build process" + ), + oldnames=["lib_use", "lib_force", "lib_install"], + multiple=True, + ), + ConfigEnvOption( + group="library", + name="lib_ignore", + description=( + "A list of library names which should be ignored by " + "Library Dependency Finder (LDF)" + ), + multiple=True, + ), + ConfigEnvOption( + group="library", + name="lib_extra_dirs", + description=( + "A list of extra directories/storages where Library Dependency " + "Finder (LDF) will look for dependencies" + ), + multiple=True, + sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS", + ), + ConfigEnvOption( + group="library", + name="lib_ldf_mode", + description=( + "Control how Library Dependency Finder (LDF) should analyze " + "dependencies (`#include` directives)" + ), + type=click.Choice(["off", "chain", "deep", "chain+", "deep+"]), + default="chain", + ), + ConfigEnvOption( + group="library", + name="lib_compat_mode", + description=( + "Configure a strictness (compatibility mode by frameworks, " + "development platforms) of Library Dependency Finder (LDF)" + ), + type=click.Choice(["off", "soft", "strict"]), + default="soft", + ), + ConfigEnvOption( + group="library", + name="lib_archive", + description=( + "Create an archive (`*.a`, static library) from the object files " + "and link it into a firmware (program)" + ), + type=click.BOOL, + default=True, + ), + # Check + ConfigEnvOption( + group="check", + name="check_tool", + description="A list of check tools used for analysis", + type=click.Choice(["cppcheck", "clangtidy", "pvs-studio"]), + multiple=True, + default=["cppcheck"], + ), + ConfigEnvOption( + group="check", + name="check_src_filters", + oldnames=["check_patterns"], + description=( + "Configure a list of target files or directories for checking" + ), + multiple=True, + ), + ConfigEnvOption( + group="check", + name="check_flags", + description="An extra flags to be passed to a check tool", + multiple=True, + ), + ConfigEnvOption( + group="check", + name="check_severity", + description="List of defect severity types for analysis", + multiple=True, + type=click.Choice(["low", "medium", "high"]), + default=["low", "medium", "high"], + ), + ConfigEnvOption( + group="check", + name="check_skip_packages", + description="Skip checking includes from packages directory", + type=click.BOOL, + default=False, + ), + # Test + ConfigEnvOption( + group="test", + name="test_framework", + description="A unit testing framework", + type=click.Choice(["doctest", "googletest", "unity", "custom"]), + default="unity", + ), + ConfigEnvOption( + group="test", + name="test_filter", + description="Process tests where the name matches specified patterns", + multiple=True, + ), + ConfigEnvOption( + group="test", + name="test_ignore", + description="Ignore tests where the name matches specified patterns", + multiple=True, + ), + ConfigEnvOption( + group="test", + name="test_port", + description="A serial port to communicate with a target device", + ), + ConfigEnvOption( + group="test", + name="test_speed", + description="A connection speed (baud rate) to communicate with " + "a target device", + type=click.INT, + default=115200, + ), + ConfigEnvOption( + group="test", + name="test_build_src", + oldnames=["test_build_project_src"], + description="Build main source code in pair with a test code", + type=click.BOOL, + default=False, + ), + ConfigEnvOption( + group="test", + name="test_testing_command", + multiple=True, + description=( + "A custom testing command that runs test cases " + "and returns results to the standard output" + ), + ), + # Debug + ConfigEnvOption( + group="debug", + name="debug_tool", + description="A name of debugging tool", + ), + ConfigEnvOption( + group="debug", + name="debug_build_flags", + description=( + "Custom debug flags/options for preprocessing, compilation, " + "assembly, and linking processes" + ), + multiple=True, + default=["-Og", "-g2", "-ggdb2"], + ), + ConfigEnvOption( + group="debug", + name="debug_init_break", + description=( + "An initial breakpoint that makes program stop whenever a " + "certain point in the program is reached" + ), + default="tbreak main", + ), + ConfigEnvOption( + group="debug", + name="debug_init_cmds", + description="Initial commands to be passed to a back-end debugger", + multiple=True, + ), + ConfigEnvOption( + group="debug", + name="debug_extra_cmds", + description="An extra commands to be passed to a back-end debugger", + multiple=True, + ), + ConfigEnvOption( + group="debug", + name="debug_load_cmds", + description=( + "A list of commands to be used to load program/firmware " + "to a target device" + ), + oldnames=["debug_load_cmd"], + multiple=True, + default=["load"], + ), + ConfigEnvOption( + group="debug", + name="debug_load_mode", + description=( + "Allows one to control when PlatformIO should load debugging " + "firmware to the end target" + ), + type=click.Choice(["always", "modified", "manual"]), + default="always", + ), + ConfigEnvOption( + group="debug", + name="debug_server", + description="Allows one to setup a custom debugging server", + multiple=True, + ), + ConfigEnvOption( + group="debug", + name="debug_port", + description=( + "A debugging port of a remote target (a serial device or " + "network address)" + ), + ), + ConfigEnvOption( + group="debug", + name="debug_speed", + description="A debug adapter speed (JTAG speed)", + ), + ConfigEnvOption( + group="debug", + name="debug_svd_path", + description=( + "A custom path to SVD file which contains information about " + "device peripherals" + ), + type=click.Path(exists=True, file_okay=True, dir_okay=False), + ), + ConfigEnvOption( + group="debug", + name="debug_server_ready_pattern", + description=( + "A pattern to determine when debugging server is ready " + "for an incoming connection" + ), + ), + ConfigEnvOption( + group="debug", + name="debug_test", + description=("A name of a unit test to be debugged"), + ), + # Advanced + ConfigEnvOption( + group="advanced", + name="extends", + description=( + "Inherit configuration from other sections or build environments" + ), + multiple=True, + ), + ConfigEnvOption( + group="advanced", + name="extra_scripts", + description="A list of PRE and POST extra scripts", + oldnames=["extra_script"], + multiple=True, + sysenvvar="PLATFORMIO_EXTRA_SCRIPTS", + ), + ] + ] +) + + +def get_config_options_schema(): + return [opt.as_dict() for opt in ProjectOptions.values()] diff --git a/platformio/project/savedeps.py b/platformio/project/savedeps.py new file mode 100644 index 0000000..1c12bd3 --- /dev/null +++ b/platformio/project/savedeps.py @@ -0,0 +1,87 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio.compat import ci_strings_are_equal +from platformio.package.meta import PackageSpec +from platformio.project.config import ProjectConfig +from platformio.project.exception import InvalidProjectConfError + + +def pkg_to_save_spec(pkg, user_spec): + assert isinstance(user_spec, PackageSpec) + if user_spec.external: + return user_spec + return PackageSpec( + owner=pkg.metadata.spec.owner, + name=pkg.metadata.spec.name, + requirements=user_spec.requirements + or ( + ("^%s" % pkg.metadata.version) + if not pkg.metadata.version.build + else pkg.metadata.version + ), + ) + + +def save_project_dependencies( + project_dir, specs, scope, action="add", environments=None +): + config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + config.expand_interpolations = False + candidates = [] + try: + candidates = _ignore_deps_by_specs(config.get("env:" + env, scope), specs) + except InvalidProjectConfError: + pass + if action == "add": + candidates.extend(spec.as_dependency() for spec in specs) + if candidates: + result = [] + for item in candidates: + item = item.strip() + if item and item not in result: + result.append(item) + config.set("env:" + env, scope, result) + elif config.has_option("env:" + env, scope): + config.remove_option("env:" + env, scope) + config.save() + + +def _ignore_deps_by_specs(deps, specs): + result = [] + for dep in deps: + ignore_conditions = [] + depspec = PackageSpec(dep) + if depspec.external: + ignore_conditions.append(depspec in specs) + else: + for spec in specs: + if depspec.owner: + ignore_conditions.append( + ci_strings_are_equal(depspec.owner, spec.owner) + and ci_strings_are_equal(depspec.name, spec.name) + ) + else: + ignore_conditions.append( + ci_strings_are_equal(depspec.name, spec.name) + ) + if not any(ignore_conditions): + result.append(dep) + return result diff --git a/platformio/public.py b/platformio/public.py new file mode 100644 index 0000000..49db1b5 --- /dev/null +++ b/platformio/public.py @@ -0,0 +1,29 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-import + +from platformio.device.list.util import list_logical_devices, list_serial_ports +from platformio.device.monitor.filters.base import DeviceMonitorFilterBase +from platformio.fs import to_unix_path +from platformio.platform.base import PlatformBase +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_watch_lib_dirs, load_build_metadata +from platformio.project.options import get_config_options_schema +from platformio.test.result import TestCase, TestCaseSource, TestStatus +from platformio.test.runners.base import TestRunnerBase +from platformio.test.runners.doctest import DoctestTestCaseParser +from platformio.test.runners.googletest import GoogletestTestRunner +from platformio.test.runners.unity import UnityTestRunner +from platformio.util import get_systype diff --git a/platformio/registry/__init__.py b/platformio/registry/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/registry/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/registry/access/__init__.py b/platformio/registry/access/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/registry/access/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/registry/access/cli.py b/platformio/registry/access/cli.py new file mode 100644 index 0000000..fe585c9 --- /dev/null +++ b/platformio/registry/access/cli.py @@ -0,0 +1,36 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.registry.access.commands.grant import access_grant_cmd +from platformio.registry.access.commands.list import access_list_cmd +from platformio.registry.access.commands.private import access_private_cmd +from platformio.registry.access.commands.public import access_public_cmd +from platformio.registry.access.commands.revoke import access_revoke_cmd + + +@click.group( + "access", + commands=[ + access_grant_cmd, + access_list_cmd, + access_private_cmd, + access_public_cmd, + access_revoke_cmd, + ], + short_help="Manage resource access", +) +def cli(): + pass diff --git a/platformio/registry/access/commands/__init__.py b/platformio/registry/access/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/registry/access/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/registry/access/commands/grant.py b/platformio/registry/access/commands/grant.py new file mode 100644 index 0000000..f921447 --- /dev/null +++ b/platformio/registry/access/commands/grant.py @@ -0,0 +1,39 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.registry.access.validate import validate_client, validate_urn +from platformio.registry.client import RegistryClient + + +@click.command("grant", short_help="Grant access") +@click.argument("level", type=click.Choice(["admin", "maintainer", "guest"])) +@click.argument( + "client", + metavar="[|]", + callback=lambda _, __, value: validate_client(value), +) +@click.argument( + "urn", + callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +def access_grant_cmd(level, client, urn, urn_type): # pylint: disable=unused-argument + reg_client = RegistryClient() + reg_client.grant_access_for_resource(urn=urn, client=client, level=level) + return click.secho( + "Access for resource %s has been granted for %s" % (urn, client), + fg="green", + ) diff --git a/platformio/registry/access/commands/list.py b/platformio/registry/access/commands/list.py new file mode 100644 index 0000000..ce9d8a6 --- /dev/null +++ b/platformio/registry/access/commands/list.py @@ -0,0 +1,58 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click +from tabulate import tabulate + +from platformio.registry.client import RegistryClient + + +@click.command("list", short_help="List published resources") +@click.argument("owner", required=False) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +@click.option("--json-output", is_flag=True) +def access_list_cmd(owner, urn_type, json_output): # pylint: disable=unused-argument + reg_client = RegistryClient() + resources = reg_client.list_resources(owner=owner) + if json_output: + return click.echo(json.dumps(resources)) + if not resources: + return click.secho("You do not have any resources.", fg="yellow") + for resource in resources: + click.echo() + click.secho(resource.get("name"), fg="cyan") + click.echo("-" * len(resource.get("name"))) + table_data = [] + table_data.append(("URN:", resource.get("urn"))) + table_data.append(("Owner:", resource.get("owner"))) + table_data.append( + ( + "Access:", + click.style("Private", fg="red") + if resource.get("private", False) + else "Public", + ) + ) + table_data.append( + ( + "Access level(s):", + ", ".join( + (level.capitalize() for level in resource.get("access_levels")) + ), + ) + ) + click.echo(tabulate(table_data, tablefmt="plain")) + return click.echo() diff --git a/platformio/registry/access/commands/private.py b/platformio/registry/access/commands/private.py new file mode 100644 index 0000000..5211aeb --- /dev/null +++ b/platformio/registry/access/commands/private.py @@ -0,0 +1,33 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.registry.access.validate import validate_urn +from platformio.registry.client import RegistryClient + + +@click.command("private", short_help="Make resource private") +@click.argument( + "urn", + callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +def access_private_cmd(urn, urn_type): # pylint: disable=unused-argument + client = RegistryClient() + client.update_resource(urn=urn, private=1) + return click.secho( + "The resource %s has been successfully updated." % urn, + fg="green", + ) diff --git a/platformio/registry/access/commands/public.py b/platformio/registry/access/commands/public.py new file mode 100644 index 0000000..465dcf3 --- /dev/null +++ b/platformio/registry/access/commands/public.py @@ -0,0 +1,33 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.registry.access.validate import validate_urn +from platformio.registry.client import RegistryClient + + +@click.command("public", short_help="Make resource public") +@click.argument( + "urn", + callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +def access_public_cmd(urn, urn_type): # pylint: disable=unused-argument + client = RegistryClient() + client.update_resource(urn=urn, private=0) + return click.secho( + "The resource %s has been successfully updated." % urn, + fg="green", + ) diff --git a/platformio/registry/access/commands/revoke.py b/platformio/registry/access/commands/revoke.py new file mode 100644 index 0000000..b6bb000 --- /dev/null +++ b/platformio/registry/access/commands/revoke.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.registry.access.validate import validate_client, validate_urn +from platformio.registry.client import RegistryClient + + +@click.command("revoke", short_help="Revoke access") +@click.argument( + "client", + metavar="[ORGNAME:TEAMNAME|USERNAME]", + callback=lambda _, __, value: validate_client(value), +) +@click.argument( + "urn", + callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +def access_revoke_cmd(client, urn, urn_type): # pylint: disable=unused-argument + reg_client = RegistryClient() + reg_client.revoke_access_from_resource(urn=urn, client=client) + return click.secho( + "Access for resource %s has been revoked for %s" % (urn, client), + fg="green", + ) diff --git a/platformio/registry/access/validate.py b/platformio/registry/access/validate.py new file mode 100644 index 0000000..71ca5a9 --- /dev/null +++ b/platformio/registry/access/validate.py @@ -0,0 +1,34 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import click + +from platformio.account.validate import validate_orgname_teamname, validate_username + + +def validate_urn(value): + value = str(value).strip() + if not re.match(r"^prn:reg:pkg:(\d+):(\w+)$", value, flags=re.I): + raise click.BadParameter("Invalid URN format.") + return value + + +def validate_client(value): + if ":" in value: + validate_orgname_teamname(value) + else: + validate_username(value) + return value diff --git a/platformio/registry/client.py b/platformio/registry/client.py new file mode 100644 index 0000000..6173f7d --- /dev/null +++ b/platformio/registry/client.py @@ -0,0 +1,162 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments + +from platformio import __registry_mirror_hosts__, fs +from platformio.account.client import AccountClient, AccountError +from platformio.http import HTTPClient, HTTPClientError + + +class RegistryClient(HTTPClient): + def __init__(self): + endpoints = [f"https://api.{host}" for host in __registry_mirror_hosts__] + super().__init__(endpoints) + + @staticmethod + def allowed_private_packages(): + private_permissions = set( + [ + "service.registry.publish-private-tool", + "service.registry.publish-private-platform", + "service.registry.publish-private-library", + ] + ) + try: + info = AccountClient().get_account_info() or {} + for item in info.get("packages", []): + if set(item.keys()) & private_permissions: + return True + except AccountError: + pass + return False + + def publish_package( # pylint: disable=redefined-builtin + self, owner, type, archive_path, released_at=None, private=False, notify=True + ): + with open(archive_path, "rb") as fp: + return self.fetch_json_data( + "post", + "/v3/packages/%s/%s" % (owner, type), + params={ + "private": 1 if private else 0, + "notify": 1 if notify else 0, + "released_at": released_at, + }, + headers={ + "Content-Type": "application/octet-stream", + "X-PIO-Content-SHA256": fs.calculate_file_hashsum( + "sha256", archive_path + ), + }, + data=fp, + x_with_authorization=True, + ) + + def unpublish_package( # pylint: disable=redefined-builtin + self, owner, type, name, version=None, undo=False + ): + path = "/v3/packages/%s/%s/%s" % (owner, type, name) + if version: + path += "/" + version + return self.fetch_json_data( + "delete", path, params={"undo": 1 if undo else 0}, x_with_authorization=True + ) + + def update_resource(self, urn, private): + return self.fetch_json_data( + "put", + "/v3/resources/%s" % urn, + data={"private": int(private)}, + x_with_authorization=True, + ) + + def grant_access_for_resource(self, urn, client, level): + return self.fetch_json_data( + "put", + "/v3/resources/%s/access" % urn, + data={"client": client, "level": level}, + x_with_authorization=True, + ) + + def revoke_access_from_resource(self, urn, client): + return self.fetch_json_data( + "delete", + "/v3/resources/%s/access" % urn, + data={"client": client}, + x_with_authorization=True, + ) + + def list_resources(self, owner): + return self.fetch_json_data( + "get", + "/v3/resources", + params={"owner": owner} if owner else None, + x_cache_valid="1h", + x_with_authorization=True, + ) + + def list_packages(self, query=None, qualifiers=None, page=None, sort=None): + search_query = [] + if qualifiers: + valid_qualifiers = ( + "authors", + "keywords", + "frameworks", + "platforms", + "headers", + "ids", + "names", + "owners", + "types", + ) + assert set(qualifiers.keys()) <= set(valid_qualifiers) + for name, values in qualifiers.items(): + for value in set( + values if isinstance(values, (list, tuple)) else [values] + ): + search_query.append('%s:"%s"' % (name[:-1], value)) + if query: + search_query.append(query) + params = dict(query=" ".join(search_query)) + if page: + params["page"] = int(page) + if sort: + params["sort"] = sort + return self.fetch_json_data( + "get", + "/v3/search", + params=params, + x_cache_valid="1h", + x_with_authorization=self.allowed_private_packages(), + ) + + def get_package(self, typex, owner, name, version=None, extra_path=None): + try: + return self.fetch_json_data( + "get", + "/v3/packages/{owner}/{type}/{name}{extra_path}".format( + type=typex, + owner=owner.lower(), + name=name.lower(), + extra_path=extra_path or "", + ), + params=dict(version=version) if version else None, + x_cache_valid="1h", + x_with_authorization=self.allowed_private_packages(), + ) + except HTTPClientError as exc: + if exc.response is not None and exc.response.status_code == 404: + return None + raise exc diff --git a/platformio/registry/mirror.py b/platformio/registry/mirror.py new file mode 100644 index 0000000..4b4508f --- /dev/null +++ b/platformio/registry/mirror.py @@ -0,0 +1,98 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from urllib.parse import urlparse + +from platformio import __registry_mirror_hosts__ +from platformio.cache import ContentCache +from platformio.http import HTTPClient +from platformio.registry.client import RegistryClient + + +class RegistryFileMirrorIterator: + HTTP_CLIENT_INSTANCES = {} + + def __init__(self, download_url): + self.download_url = download_url + self._url_parts = urlparse(download_url) + self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) + self._visited_mirrors = [] + + def __iter__(self): # pylint: disable=non-iterator-returned + return self + + def __next__(self): + cache_key = ContentCache.key_from_args( + "head", self.download_url, self._visited_mirrors + ) + with ContentCache("http") as cc: + result = cc.get(cache_key) + if result is not None: + try: + headers = json.loads(result) + return ( + headers["Location"], + headers["X-PIO-Content-SHA256"], + ) + except (ValueError, KeyError): + pass + + http = self.get_http_client() + response = http.send_request( + "head", + self._url_parts.path, + allow_redirects=False, + params=dict(bypass=",".join(self._visited_mirrors)) + if self._visited_mirrors + else None, + x_with_authorization=RegistryClient.allowed_private_packages(), + ) + stop_conditions = [ + response.status_code not in (302, 307), + not response.headers.get("Location"), + not response.headers.get("X-PIO-Mirror"), + response.headers.get("X-PIO-Mirror") in self._visited_mirrors, + ] + if any(stop_conditions): + raise StopIteration + self._visited_mirrors.append(response.headers.get("X-PIO-Mirror")) + cc.set( + cache_key, + json.dumps( + { + "Location": response.headers.get("Location"), + "X-PIO-Content-SHA256": response.headers.get( + "X-PIO-Content-SHA256" + ), + } + ), + "1h", + ) + return ( + response.headers.get("Location"), + response.headers.get("X-PIO-Content-SHA256"), + ) + + def get_http_client(self): + if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: + endpoints = [self._mirror] + for host in __registry_mirror_hosts__: + endpoint = f"https://dl.{host}" + if endpoint not in endpoints: + endpoints.append(endpoint) + RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( + endpoints + ) + return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] diff --git a/platformio/remote/__init__.py b/platformio/remote/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/remote/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/remote/ac/__init__.py b/platformio/remote/ac/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/remote/ac/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/remote/ac/base.py b/platformio/remote/ac/base.py new file mode 100644 index 0000000..0c631c6 --- /dev/null +++ b/platformio/remote/ac/base.py @@ -0,0 +1,90 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + + +class AsyncCommandBase: + MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb + + def __init__(self, options=None, on_end_callback=None): + self.options = options or {} + self.on_end_callback = on_end_callback + self._buffer = b"" + self._return_code = None + self._d = None + self._paused = False + + try: + self.start() + except Exception as exc: + raise pb.Error(str(exc)) from exc + + @property + def id(self): + return id(self) + + def pause(self): + self._paused = True + self.stop() + + def unpause(self): + self._paused = False + self.start() + + def start(self): + raise NotImplementedError + + def stop(self): + self.transport.loseConnection() # pylint: disable=no-member + + def _ac_ended(self): + if self.on_end_callback: + self.on_end_callback() + if not self._d or self._d.called: + self._d = None + return + if self._buffer: + self._d.callback(self._buffer) + else: + self._d.callback(None) + + def _ac_ondata(self, data): + self._buffer += data + if len(self._buffer) > self.MAX_BUFFER_SIZE: + self._buffer = self._buffer[-1 * self.MAX_BUFFER_SIZE :] + if self._paused: + return + if self._d and not self._d.called: + self._d.callback(self._buffer) + self._buffer = b"" + + def ac_read(self): + if self._buffer: + result = self._buffer + self._buffer = b"" + return result + if self._return_code is None: + self._d = defer.Deferred() + return self._d + return None + + def ac_write(self, data): + self.transport.write(data) # pylint: disable=no-member + return len(data) + + def ac_close(self): + self.stop() + return self._return_code diff --git a/platformio/remote/ac/process.py b/platformio/remote/ac/process.py new file mode 100644 index 0000000..91da456 --- /dev/null +++ b/platformio/remote/ac/process.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from twisted.internet import protocol, reactor # pylint: disable=import-error + +from platformio.remote.ac.base import AsyncCommandBase + + +class ProcessAsyncCmd(protocol.ProcessProtocol, AsyncCommandBase): + def start(self): + env = dict(os.environ).copy() + env.update({"PLATFORMIO_FORCE_ANSI": "true"}) + reactor.spawnProcess( + self, self.options["executable"], self.options["args"], env + ) + + def outReceived(self, data): + self._ac_ondata(data) + + def errReceived(self, data): + self._ac_ondata(data) + + def processExited(self, reason): + self._return_code = reason.value.exitCode + + def processEnded(self, reason): + if self._return_code is None: + self._return_code = reason.value.exitCode + self._ac_ended() diff --git a/platformio/remote/ac/psync.py b/platformio/remote/ac/psync.py new file mode 100644 index 0000000..8728737 --- /dev/null +++ b/platformio/remote/ac/psync.py @@ -0,0 +1,66 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import zlib +from io import BytesIO + +from platformio.remote.ac.base import AsyncCommandBase +from platformio.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync + + +class ProjectSyncAsyncCmd(AsyncCommandBase): + def __init__(self, *args, **kwargs): + self.psync = None + self._upstream = None + super().__init__(*args, **kwargs) + + def start(self): + project_dir = os.path.join( + self.options["agent_working_dir"], "projects", self.options["id"] + ) + self.psync = ProjectSync(project_dir) + for name in self.options["items"]: + self.psync.add_item(os.path.join(project_dir, name), name) + + def stop(self): + self.psync = None + self._upstream = None + self._return_code = PROJECT_SYNC_STAGE.COMPLETED.value + + def ac_write(self, data): + stage = PROJECT_SYNC_STAGE.lookupByValue(data.get("stage")) + + if stage is PROJECT_SYNC_STAGE.DBINDEX: + self.psync.rebuild_dbindex() + return zlib.compress(json.dumps(self.psync.get_dbindex()).encode()) + + if stage is PROJECT_SYNC_STAGE.DELETE: + return self.psync.delete_dbindex( + json.loads(zlib.decompress(data["dbindex"])) + ) + + if stage is PROJECT_SYNC_STAGE.UPLOAD: + if not self._upstream: + self._upstream = BytesIO() + self._upstream.write(data["chunk"]) + if self._upstream.tell() == data["total"]: + self.psync.decompress_items(self._upstream) + self._upstream = None + return PROJECT_SYNC_STAGE.EXTRACTED.value + + return PROJECT_SYNC_STAGE.UPLOAD.value + + return None diff --git a/platformio/remote/ac/serial.py b/platformio/remote/ac/serial.py new file mode 100644 index 0000000..7f7f83f --- /dev/null +++ b/platformio/remote/ac/serial.py @@ -0,0 +1,60 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + +from twisted.internet import protocol, reactor # pylint: disable=import-error +from twisted.internet.serialport import SerialPort # pylint: disable=import-error + +from platformio.remote.ac.base import AsyncCommandBase + + +class SerialPortAsyncCmd(protocol.Protocol, AsyncCommandBase): + def start(self): + SerialPort( + self, + reactor=reactor, + **{ + "deviceNameOrPortNumber": self.options["port"], + "baudrate": self.options["baud"], + "parity": self.options["parity"], + "rtscts": 1 if self.options["rtscts"] else 0, + "xonxoff": 1 if self.options["xonxoff"] else 0, + } + ) + + def connectionMade(self): + self.reset_device() + if self.options.get("rts", None) is not None: + self.transport.setRTS(self.options.get("rts")) + if self.options.get("dtr", None) is not None: + self.transport.setDTR(self.options.get("dtr")) + + def reset_device(self): + self.transport.flushInput() + self.transport.setDTR(False) + self.transport.setRTS(False) + sleep(0.1) + self.transport.setDTR(True) + self.transport.setRTS(True) + sleep(0.1) + + def dataReceived(self, data): + self._ac_ondata(data) + + def connectionLost(self, reason): # pylint: disable=unused-argument + if self._paused: + return + self._return_code = 0 + self._ac_ended() diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py new file mode 100644 index 0000000..f1775d4 --- /dev/null +++ b/platformio/remote/cli.py @@ -0,0 +1,412 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments, import-outside-toplevel +# pylint: disable=inconsistent-return-statements + +import os +import subprocess +import sys +import threading +from site import addsitedir +from tempfile import mkdtemp +from time import sleep + +import click + +from platformio import fs, proc +from platformio.device.monitor.command import ( + apply_project_monitor_options, + device_monitor_cmd, + get_project_options, +) +from platformio.package.manager.core import get_core_package_dir +from platformio.project.exception import NotPlatformIOProjectError +from platformio.project.options import ProjectOptions +from platformio.run.cli import cli as cmd_run +from platformio.test.cli import cli as test_cmd + + +@click.group("remote", short_help="Remote Development") +@click.option("-a", "--agent", multiple=True) +@click.pass_context +def cli(ctx, agent): + ctx.obj = agent + # inject twisted dependencies + contrib_dir = get_core_package_dir("contrib-pioremote") + if contrib_dir not in sys.path: + addsitedir(contrib_dir) + sys.path.insert(0, contrib_dir) + + +@cli.group("agent", short_help="Start a new agent or list active") +def remote_agent(): + pass + + +@remote_agent.command("start", short_help="Start agent") +@click.option("-n", "--name") +@click.option("-s", "--share", multiple=True, metavar="E-MAIL") +@click.option( + "-d", + "--working-dir", + envvar="PLATFORMIO_REMOTE_AGENT_DIR", + type=click.Path(file_okay=False, dir_okay=True, writable=True), +) +def remote_agent_start(name, share, working_dir): + from platformio.remote.client.agent_service import RemoteAgentService + + RemoteAgentService(name, share, working_dir).connect() + + +@remote_agent.command("list", short_help="List active agents") +def remote_agent_list(): + from platformio.remote.client.agent_list import AgentListClient + + AgentListClient().connect() + + +@cli.command("update", short_help="Update installed Platforms, Packages and Libraries") +@click.option( + "-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead", +) +@click.option( + "--dry-run", is_flag=True, help="Do not update, only check for the new versions" +) +@click.pass_obj +def remote_update(agents, only_check, dry_run): + from platformio.remote.client.update_core import UpdateCoreClient + + UpdateCoreClient("update", agents, dict(only_check=only_check or dry_run)).connect() + + +@cli.command("run", short_help="Process project environments remotely") +@click.option("-e", "--environment", multiple=True) +@click.option("-t", "--target", multiple=True) +@click.option("--upload-port") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), +) +@click.option("--disable-auto-clean", is_flag=True) +@click.option("-r", "--force-remote", is_flag=True) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.pass_obj +@click.pass_context +def remote_run( + ctx, + agents, + environment, + target, + upload_port, + project_dir, + disable_auto_clean, + force_remote, + silent, + verbose, +): + from platformio.remote.client.run_or_test import RunOrTestClient + + cr = RunOrTestClient( + "run", + agents, + dict( + environment=environment, + target=target, + upload_port=upload_port, + project_dir=project_dir, + disable_auto_clean=disable_auto_clean, + force_remote=force_remote, + silent=silent, + verbose=verbose, + ), + ) + if force_remote: + return cr.connect() + + click.secho("Building project locally", bold=True) + local_targets = [] + if "clean" in target: + local_targets = ["clean"] + elif set(["buildfs", "uploadfs", "uploadfsota"]) & set(target): + local_targets = ["buildfs"] + else: + local_targets = ["checkprogsize", "buildprog"] + ctx.invoke( + cmd_run, + environment=environment, + target=local_targets, + project_dir=project_dir, + # disable_auto_clean=True, + silent=silent, + verbose=verbose, + ) + + if any(["upload" in t for t in target] + ["program" in target]): + click.secho("Uploading firmware remotely", bold=True) + cr.options["target"] += ("nobuild",) + cr.options["disable_auto_clean"] = True + cr.connect() + + return True + + +@cli.command("test", short_help="Remote Unit Testing") +@click.option("--environment", "-e", multiple=True, metavar="") +@click.option( + "--filter", + "-f", + multiple=True, + metavar="", + help="Filter tests by a pattern", +) +@click.option( + "--ignore", + "-i", + multiple=True, + metavar="", + help="Ignore tests by a pattern", +) +@click.option("--upload-port") +@click.option("--test-port") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), +) +@click.option("-r", "--force-remote", is_flag=True) +@click.option("--without-building", is_flag=True) +@click.option("--without-uploading", is_flag=True) +@click.option("--verbose", "-v", is_flag=True) +@click.pass_obj +@click.pass_context +def remote_test( # pylint: disable=redefined-builtin + ctx, + agents, + environment, + filter, + ignore, + upload_port, + test_port, + project_dir, + force_remote, + without_building, + without_uploading, + verbose, +): + from platformio.remote.client.run_or_test import RunOrTestClient + + cr = RunOrTestClient( + "test", + agents, + dict( + environment=environment, + filter=filter, + ignore=ignore, + upload_port=upload_port, + test_port=test_port, + project_dir=project_dir, + force_remote=force_remote, + without_building=without_building, + without_uploading=without_uploading, + verbose=verbose, + ), + ) + if force_remote: + return cr.connect() + + click.secho("Building project locally", bold=True) + + ctx.invoke( + test_cmd, + environment=environment, + filter=filter, + ignore=ignore, + project_dir=project_dir, + without_uploading=True, + without_testing=True, + verbose=verbose, + ) + + click.secho("Testing project remotely", bold=True) + cr.options["without_building"] = True + cr.connect() + + return True + + +@cli.group("device", short_help="Monitor remote device or list existing") +def remote_device(): + pass + + +@remote_device.command("list", short_help="List remote devices") +@click.option("--json-output", is_flag=True) +@click.pass_obj +def device_list(agents, json_output): + from platformio.remote.client.device_list import DeviceListClient + + DeviceListClient(agents, json_output).connect() + + +@remote_device.command("monitor", short_help="Monitor remote device") +@click.option("--port", "-p", help="Port, a number or a device name") +@click.option( + "-b", + "--baud", + type=ProjectOptions["env.monitor_speed"].type, + help="Set baud/speed [default=%d]" % ProjectOptions["env.monitor_speed"].default, +) +@click.option( + "--parity", + type=ProjectOptions["env.monitor_parity"].type, + help="Set parity [default=%s]" % ProjectOptions["env.monitor_parity"].default, +) +@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control") +@click.option("--xonxoff", is_flag=True, help="Enable software flow control") +@click.option( + "--rts", + type=ProjectOptions["env.monitor_rts"].type, + help="Set initial RTS line state", +) +@click.option( + "--dtr", + type=ProjectOptions["env.monitor_dtr"].type, + help="Set initial DTR line state", +) +@click.option("--echo", is_flag=True, help="Enable local echo") +@click.option( + "--encoding", + default="UTF-8", + show_default=True, + help="Set the encoding for the serial port (e.g. hexlify, Latin1, UTF-8)", +) +@click.option( + "-f", + "--filter", + "filters", + multiple=True, + help="Apply filters/text transformations", +) +@click.option( + "--eol", + type=ProjectOptions["env.monitor_eol"].type, + help="End of line mode [default=%s]" % ProjectOptions["env.monitor_eol"].default, +) +@click.option("--raw", is_flag=True, help=ProjectOptions["env.monitor_raw"].description) +@click.option( + "--exit-char", + type=int, + default=3, + show_default=True, + help="ASCII code of special character that is used to exit " + "the application [default=3 (Ctrl+C)]", +) +@click.option( + "--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control terminal (menu) [default=20 (DEC)]", +) +@click.option( + "--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages", +) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option( + "-e", + "--environment", + help="Load configuration from `platformio.ini` and specified environment", +) +@click.option( + "--sock", + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), +) +@click.pass_obj +@click.pass_context +def device_monitor(ctx, agents, **kwargs): + from platformio.remote.client.device_monitor import DeviceMonitorClient + + if kwargs["sock"]: + return DeviceMonitorClient(agents, **kwargs).connect() + + project_options = {} + try: + with fs.cd(kwargs["project_dir"]): + project_options = get_project_options(kwargs["environment"]) + except NotPlatformIOProjectError: + pass + + kwargs = apply_project_monitor_options(kwargs, project_options) + + def _tx_target(sock_dir): + subcmd_argv = ["remote"] + for agent in agents: + subcmd_argv.extend(["--agent", agent]) + subcmd_argv.extend(["device", "monitor"]) + subcmd_argv.extend(project_options_to_monitor_argv(kwargs)) + subcmd_argv.extend(["--sock", sock_dir]) + subprocess.call([proc.where_is_program("platformio")] + subcmd_argv) + + sock_dir = mkdtemp(suffix="pio") + sock_file = os.path.join(sock_dir, "sock") + try: + t = threading.Thread(target=_tx_target, args=(sock_dir,)) + t.start() + while t.is_alive() and not os.path.isfile(sock_file): + sleep(0.1) + if not t.is_alive(): + return + with open(sock_file, encoding="utf8") as fp: + kwargs["port"] = fp.read() + kwargs["no_reconnect"] = True + ctx.invoke(device_monitor_cmd, **kwargs) + t.join(2) + finally: + fs.rmtree(sock_dir) + + return True + + +def project_options_to_monitor_argv(cli_options): + result = [] + for item in cli_options["filters"] or []: + result.extend(["--filter", item]) + for k, v in cli_options.items(): + if v is None or k == "filters": + continue + k = "--" + k.replace("_", "-") + if isinstance(v, bool): + if v: + result.append(k) + elif isinstance(v, tuple): + for i in v: + result.extend([k, i]) + else: + result.extend([k, str(v)]) + return result diff --git a/platformio/remote/client/__init__.py b/platformio/remote/client/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/remote/client/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/remote/client/agent_list.py b/platformio/remote/client/agent_list.py new file mode 100644 index 0000000..103c0f8 --- /dev/null +++ b/platformio/remote/client/agent_list.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +import click + +from platformio.remote.client.base import RemoteClientBase + + +class AgentListClient(RemoteClientBase): + def agent_pool_ready(self): + d = self.agentpool.callRemote("list", True) + d.addCallback(self._cbResult) + d.addErrback(self.cb_global_error) + + def _cbResult(self, result): + for item in result: + click.secho(item["name"], fg="cyan") + click.echo("-" * len(item["name"])) + click.echo("ID: %s" % item["id"]) + click.echo( + "Started: %s" + % datetime.fromtimestamp(item["started"]).strftime("%Y-%m-%d %H:%M:%S") + ) + click.echo("") + self.disconnect() diff --git a/platformio/remote/client/agent_service.py b/platformio/remote/client/agent_service.py new file mode 100644 index 0000000..3a884e6 --- /dev/null +++ b/platformio/remote/client/agent_service.py @@ -0,0 +1,228 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from twisted.logger import LogLevel # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio import proc +from platformio.device.list.util import list_serial_ports +from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError +from platformio.remote.ac.process import ProcessAsyncCmd +from platformio.remote.ac.psync import ProjectSyncAsyncCmd +from platformio.remote.ac.serial import SerialPortAsyncCmd +from platformio.remote.client.base import RemoteClientBase + + +class RemoteAgentService(RemoteClientBase): + def __init__(self, name, share, working_dir=None): + RemoteClientBase.__init__(self) + self.log_level = LogLevel.info + self.working_dir = working_dir or os.path.join( + ProjectConfig.get_instance().get("platformio", "core_dir"), "remote" + ) + if not os.path.isdir(self.working_dir): + os.makedirs(self.working_dir) + if name: + self.name = str(name)[:50] + self.join_options.update( + {"agent": True, "share": [s.lower().strip()[:50] for s in share]} + ) + + self._acs = {} + + def agent_pool_ready(self): + pass + + def cb_disconnected(self, reason): + for ac in self._acs.values(): + ac.ac_close() + RemoteClientBase.cb_disconnected(self, reason) + + def remote_acread(self, ac_id): + self.log.debug("Async Read: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return self._acs[ac_id].ac_read() + + def remote_acwrite(self, ac_id, data): + self.log.debug("Async Write: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return self._acs[ac_id].ac_write(data) + + def remote_acclose(self, ac_id): + self.log.debug("Async Close: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return_code = self._acs[ac_id].ac_close() + del self._acs[ac_id] + return return_code + + def remote_cmd(self, cmd, options): + self.log.info("Remote command received: {cmd}", cmd=cmd) + self.log.debug("Command options: {options!r}", options=options) + callback = "_process_cmd_%s" % cmd.replace(".", "_") + return getattr(self, callback)(options) + + def _defer_async_cmd(self, ac, pass_agent_name=True): + self._acs[ac.id] = ac + if pass_agent_name: + return (self.id, ac.id, self.name) + return (self.id, ac.id) + + def _process_cmd_device_list(self, _): + return (self.name, list_serial_ports()) + + def _process_cmd_device_monitor(self, options): + if not options["port"]: + for item in list_serial_ports(): + if "VID:PID" in item["hwid"]: + options["port"] = item["port"] + break + + # terminate opened monitors + if options["port"]: + for ac in list(self._acs.values()): + if ( + isinstance(ac, SerialPortAsyncCmd) + and ac.options["port"] == options["port"] + ): + self.log.info( + "Terminate previously opened monitor at {port}", + port=options["port"], + ) + ac.ac_close() + del self._acs[ac.id] + + if not options["port"]: + raise pb.Error("Please specify serial port using `--port` option") + self.log.info("Starting serial monitor at {port}", port=options["port"]) + + return self._defer_async_cmd(SerialPortAsyncCmd(options), pass_agent_name=False) + + def _process_cmd_psync(self, options): + for ac in list(self._acs.values()): + if ( + isinstance(ac, ProjectSyncAsyncCmd) + and ac.options["id"] == options["id"] + ): + self.log.info("Terminate previous Project Sync process") + ac.ac_close() + del self._acs[ac.id] + + options["agent_working_dir"] = self.working_dir + return self._defer_async_cmd( + ProjectSyncAsyncCmd(options), pass_agent_name=False + ) + + def _process_cmd_run(self, options): + return self._process_cmd_run_or_test("run", options) + + def _process_cmd_test(self, options): + return self._process_cmd_run_or_test("test", options) + + def _process_cmd_run_or_test( # pylint: disable=too-many-locals,too-many-branches,too-many-statements + self, command, options + ): + assert options and "project_id" in options + project_dir = os.path.join(self.working_dir, "projects", options["project_id"]) + origin_pio_ini = os.path.join(project_dir, "platformio.ini") + back_pio_ini = os.path.join(project_dir, "platformio.ini.bak") + + # remove insecure project options + try: + conf = ProjectConfig(origin_pio_ini) + if os.path.isfile(back_pio_ini): + os.remove(back_pio_ini) + os.rename(origin_pio_ini, back_pio_ini) + # cleanup + if conf.has_section("platformio"): + for opt in conf.options("platformio"): + if opt.endswith("_dir"): + conf.remove_option("platformio", opt) + else: + conf.add_section("platformio") + conf.set("platformio", "build_dir", ".pio/build") + conf.save(origin_pio_ini) + + # restore A/M times + os.utime( + origin_pio_ini, + (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)), + ) + except NotPlatformIOProjectError as exc: + raise pb.Error(str(exc)) from exc + + cmd_args = ["platformio", "--force", command, "-d", project_dir] + for env in options.get("environment", []): + cmd_args.extend(["-e", env]) + for target in options.get("target", []): + cmd_args.extend(["-t", target]) + for filter_ in options.get("filter", []): + cmd_args.extend(["-f", filter_]) + for ignore in options.get("ignore", []): + cmd_args.extend(["-i", ignore]) + if options.get("upload_port", False): + cmd_args.extend(["--upload-port", options.get("upload_port")]) + if options.get("test_port", False): + cmd_args.extend(["--test-port", options.get("test_port")]) + if options.get("disable_auto_clean", False): + cmd_args.append("--disable-auto-clean") + if options.get("without_building", False): + cmd_args.append("--without-building") + if options.get("without_uploading", False): + cmd_args.append("--without-uploading") + if options.get("silent", False): + cmd_args.append("-s") + if options.get("verbose", False): + cmd_args.append("-v") + + paused_acs = [] + for ac in self._acs.values(): + if not isinstance(ac, SerialPortAsyncCmd): + continue + self.log.info("Pause active monitor at {port}", port=ac.options["port"]) + ac.pause() + paused_acs.append(ac) + + def _cb_on_end(): + if os.path.isfile(back_pio_ini): + if os.path.isfile(origin_pio_ini): + os.remove(origin_pio_ini) + os.rename(back_pio_ini, origin_pio_ini) + for ac in paused_acs: + ac.unpause() + self.log.info( + "Unpause active monitor at {port}", port=ac.options["port"] + ) + + return self._defer_async_cmd( + ProcessAsyncCmd( + {"executable": proc.where_is_program("platformio"), "args": cmd_args}, + on_end_callback=_cb_on_end, + ) + ) + + def _process_cmd_update(self, options): + cmd_args = ["platformio", "--force", "update"] + if options.get("only_check"): + cmd_args.append("--only-check") + return self._defer_async_cmd( + ProcessAsyncCmd( + {"executable": proc.where_is_program("platformio"), "args": cmd_args} + ) + ) diff --git a/platformio/remote/client/async_base.py b/platformio/remote/client/async_base.py new file mode 100644 index 0000000..2a67264 --- /dev/null +++ b/platformio/remote/client/async_base.py @@ -0,0 +1,65 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +from twisted.spread import pb # pylint: disable=import-error + +from platformio.remote.client.base import RemoteClientBase + + +class AsyncClientBase(RemoteClientBase): + def __init__(self, command, agents, options): + RemoteClientBase.__init__(self) + self.command = command + self.agents = agents + self.options = options + + self._acs_total = 0 + self._acs_ended = 0 + + def agent_pool_ready(self): + pass + + def cb_async_result(self, result): + if self._acs_total == 0: + self._acs_total = len(result) + for success, value in result: + if not success: + raise pb.Error(value) + self.acread_data(*value) + + def acread_data(self, agent_id, ac_id, agent_name=None): + d = self.agentpool.callRemote("acread", agent_id, ac_id) + d.addCallback(self.cb_acread_result, agent_id, ac_id, agent_name) + d.addErrback(self.cb_global_error) + + def cb_acread_result(self, result, agent_id, ac_id, agent_name): + if result is None: + self.acclose(agent_id, ac_id) + else: + if self._acs_total > 1 and agent_name: + click.echo("[%s] " % agent_name, nl=False) + click.echo(result, nl=False) + self.acread_data(agent_id, ac_id, agent_name) + + def acclose(self, agent_id, ac_id): + d = self.agentpool.callRemote("acclose", agent_id, ac_id) + d.addCallback(self.cb_acclose_result) + d.addErrback(self.cb_global_error) + + def cb_acclose_result(self, exit_code): + self._acs_ended += 1 + if self._acs_ended != self._acs_total: + return + self.disconnect(exit_code) diff --git a/platformio/remote/client/base.py b/platformio/remote/client/base.py new file mode 100644 index 0000000..cf5c240 --- /dev/null +++ b/platformio/remote/client/base.py @@ -0,0 +1,192 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from time import time + +import click +from twisted.internet import defer, endpoints, reactor # pylint: disable=import-error +from twisted.logger import ILogObserver # pylint: disable=import-error +from twisted.logger import Logger # pylint: disable=import-error +from twisted.logger import LogLevel # pylint: disable=import-error +from twisted.logger import formatEvent # pylint: disable=import-error +from twisted.python import failure # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error +from zope.interface import provider # pylint: disable=import-error + +from platformio import __pioremote_endpoint__, __version__, app, exception, maintenance +from platformio.remote.factory.client import RemoteClientFactory +from platformio.remote.factory.ssl import SSLContextFactory + + +class RemoteClientBase( # pylint: disable=too-many-instance-attributes + pb.Referenceable +): + PING_DELAY = 60 + PING_MAX_FAILURES = 3 + DEBUG = False + + def __init__(self): + self.log_level = LogLevel.warn + self.log = Logger(namespace="remote", observer=self._log_observer) + self.id = app.get_host_id() + self.name = app.get_host_name() + self.join_options = {"corever": __version__} + self.perspective = None + self.agentpool = None + + self._ping_id = 0 + self._ping_caller = None + self._ping_counter = 0 + self._reactor_stopped = False + self._exit_code = 0 + + @provider(ILogObserver) + def _log_observer(self, event): + if not self.DEBUG and ( + event["log_namespace"] != self.log.namespace + or self.log_level > event["log_level"] + ): + return + msg = formatEvent(event) + click.echo( + "%s [%s] %s" + % ( + datetime.fromtimestamp(event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"), + event["log_level"].name, + msg, + ) + ) + + def connect(self): + self.log.info("Name: {name}", name=self.name) + self.log.info("Connecting to PlatformIO Remote Development Cloud") + + # pylint: disable=protected-access + proto, options = endpoints._parse(__pioremote_endpoint__) + proto = proto[0] + + factory = RemoteClientFactory() + factory.remote_client = self + factory.sslContextFactory = None + if proto == "ssl": + factory.sslContextFactory = SSLContextFactory(options["host"]) + reactor.connectSSL( + options["host"], + int(options["port"]), + factory, + factory.sslContextFactory, + ) + elif proto == "tcp": + reactor.connectTCP(options["host"], int(options["port"]), factory) + else: + raise exception.PlatformioException("Unknown PIO Remote Cloud protocol") + reactor.run() + + if self._exit_code != 0: + raise exception.ReturnErrorCode(self._exit_code) + + def cb_client_authorization_failed(self, err): + msg = "Bad account credentials" + if err.check(pb.Error): + msg = err.getErrorMessage() + self.log.error(msg) + self.disconnect(exit_code=1) + + def cb_client_authorization_made(self, perspective): + self.log.info("Successfully authorized") + self.perspective = perspective + d = perspective.callRemote("join", self.id, self.name, self.join_options) + d.addCallback(self._cb_client_join_made) + d.addErrback(self.cb_global_error) + + def _cb_client_join_made(self, result): + code = result[0] + if code == 1: + self.agentpool = result[1] + self.agent_pool_ready() + self.restart_ping() + elif code == 2: + self.remote_service(*result[1:]) + + def remote_service(self, command, options): + if command == "disconnect": + self.log.error( + "PIO Remote Cloud disconnected: {msg}", msg=options.get("message") + ) + self.disconnect() + + def restart_ping(self, reset_counter=True): + # stop previous ping callers + self.stop_ping(reset_counter) + self._ping_caller = reactor.callLater(self.PING_DELAY, self._do_ping) + + def _do_ping(self): + self._ping_counter += 1 + self._ping_id = int(time()) + d = self.perspective.callRemote("service", "ping", {"id": self._ping_id}) + d.addCallback(self._cb_pong) + d.addErrback(self._cb_pong) + + def stop_ping(self, reset_counter=True): + if reset_counter: + self._ping_counter = 0 + if not self._ping_caller or not self._ping_caller.active(): + return + self._ping_caller.cancel() + self._ping_caller = None + + def _cb_pong(self, result): + if not isinstance(result, failure.Failure) and self._ping_id == result: + self.restart_ping() + return + if self._ping_counter >= self.PING_MAX_FAILURES: + self.stop_ping() + self.perspective.broker.transport.loseConnection() + else: + self.restart_ping(reset_counter=False) + + def agent_pool_ready(self): + raise NotImplementedError + + def disconnect(self, exit_code=None): + self.stop_ping() + if exit_code is not None: + self._exit_code = exit_code + if reactor.running and not self._reactor_stopped: + self._reactor_stopped = True + reactor.stop() + + def cb_disconnected(self, _): + self.stop_ping() + self.perspective = None + self.agentpool = None + + def cb_global_error(self, err): + if err.check(pb.PBConnectionLost, defer.CancelledError): + return + + msg = err.getErrorMessage() + if err.check(pb.DeadReferenceError): + msg = "Remote Client has been terminated" + elif "PioAgentNotStartedError" in str(err.type): + msg = ( + "Could not find active agents. Please start it before on " + "a remote machine using `pio remote agent start` command.\n" + "See http://docs.platformio.org/page/plus/pio-remote.html" + ) + else: + maintenance.on_platformio_exception(Exception(err.type)) + click.secho(msg, fg="red", err=True) + self.disconnect(exit_code=1) diff --git a/platformio/remote/client/device_list.py b/platformio/remote/client/device_list.py new file mode 100644 index 0000000..f05bbfd --- /dev/null +++ b/platformio/remote/client/device_list.py @@ -0,0 +1,54 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click + +from platformio.remote.client.base import RemoteClientBase + + +class DeviceListClient(RemoteClientBase): + def __init__(self, agents, json_output): + RemoteClientBase.__init__(self) + self.agents = agents + self.json_output = json_output + + def agent_pool_ready(self): + d = self.agentpool.callRemote("cmd", self.agents, "device.list") + d.addCallback(self._cbResult) + d.addErrback(self.cb_global_error) + + def _cbResult(self, result): + data = {} + for success, value in result: + if not success: + click.secho(value, fg="red", err=True) + continue + (agent_name, devlist) = value + data[agent_name] = devlist + + if self.json_output: + click.echo(json.dumps(data)) + else: + for agent_name, devlist in data.items(): + click.echo("Agent %s" % click.style(agent_name, fg="cyan", bold=True)) + click.echo("=" * (6 + len(agent_name))) + for item in devlist: + click.secho(item["port"], fg="cyan") + click.echo("-" * len(item["port"])) + click.echo("Hardware ID: %s" % item["hwid"]) + click.echo("Description: %s" % item["description"]) + click.echo("") + self.disconnect() diff --git a/platformio/remote/client/device_monitor.py b/platformio/remote/client/device_monitor.py new file mode 100644 index 0000000..1499dd5 --- /dev/null +++ b/platformio/remote/client/device_monitor.py @@ -0,0 +1,239 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from fnmatch import fnmatch + +import click +from twisted.internet import protocol, reactor, task # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio.remote.client.base import RemoteClientBase + + +class SMBridgeProtocol(protocol.Protocol): + def connectionMade(self): + self.factory.add_client(self) + + def connectionLost(self, reason): # pylint: disable=unused-argument + self.factory.remove_client(self) + + def dataReceived(self, data): + self.factory.send_to_server(data) + + +class SMBridgeFactory(protocol.ServerFactory): + def __init__(self, cdm): + self.cdm = cdm + self._clients = [] + + def buildProtocol(self, addr): # pylint: disable=unused-argument + p = SMBridgeProtocol() + p.factory = self # pylint: disable=attribute-defined-outside-init + return p + + def add_client(self, client): + self.cdm.log.debug("SMBridge: Client connected") + self._clients.append(client) + self.cdm.acread_data() + + def remove_client(self, client): + self.cdm.log.debug("SMBridge: Client disconnected") + self._clients.remove(client) + if not self._clients: + self.cdm.client_terminal_stopped() + + def has_clients(self): + return len(self._clients) + + def send_to_clients(self, data): + if not self._clients: + return None + for client in self._clients: + client.transport.write(data) + return len(data) + + def send_to_server(self, data): + self.cdm.acwrite_data(data) + + +class DeviceMonitorClient( # pylint: disable=too-many-instance-attributes + RemoteClientBase +): + MAX_BUFFER_SIZE = 1024 * 1024 + + def __init__(self, agents, **kwargs): + RemoteClientBase.__init__(self) + self.agents = agents + self.cmd_options = kwargs + + self._bridge_factory = SMBridgeFactory(self) + self._agent_id = None + self._ac_id = None + self._d_acread = None + self._d_acwrite = None + self._acwrite_buffer = b"" + + def agent_pool_ready(self): + d = task.deferLater( + reactor, 1, self.agentpool.callRemote, "cmd", self.agents, "device.list" + ) + d.addCallback(self._cb_device_list) + d.addErrback(self.cb_global_error) + + def _cb_device_list(self, result): + devices = [] + hwid_devindexes = [] + for success, value in result: + if not success: + click.secho(value, fg="red", err=True) + continue + (agent_name, ports) = value + for item in ports: + if "VID:PID" in item["hwid"]: + hwid_devindexes.append(len(devices)) + devices.append((agent_name, item)) + + if len(result) == 1 and self.cmd_options["port"]: + if set(["*", "?", "[", "]"]) & set(self.cmd_options["port"]): + for agent, item in devices: + if fnmatch(item["port"], self.cmd_options["port"]): + return self.start_remote_monitor(agent, item["port"]) + return self.start_remote_monitor(result[0][1][0], self.cmd_options["port"]) + + device = None + if len(hwid_devindexes) == 1: + device = devices[hwid_devindexes[0]] + else: + click.echo("Available ports:") + for i, device in enumerate(devices): + click.echo( + "{index}. {host}{port} \t{description}".format( + index=i + 1, + host=device[0] + ":" if len(result) > 1 else "", + port=device[1]["port"], + description=device[1]["description"] + if device[1]["description"] != "n/a" + else "", + ) + ) + device_index = click.prompt( + "Please choose a port (number in the list above)", + type=click.Choice([str(i + 1) for i, _ in enumerate(devices)]), + ) + device = devices[int(device_index) - 1] + + self.start_remote_monitor(device[0], device[1]["port"]) + + return None + + def start_remote_monitor(self, agent, port): + options = {"port": port} + for key in ("baud", "parity", "rtscts", "xonxoff", "rts", "dtr"): + options[key] = self.cmd_options[key] + + click.echo( + "Starting Serial Monitor on {host}:{port}".format( + host=agent, port=options["port"] + ) + ) + d = self.agentpool.callRemote("cmd", [agent], "device.monitor", options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) + + def cb_async_result(self, result): + if len(result) != 1: + raise pb.Error("Invalid response from Remote Cloud") + success, value = result[0] + if not success: + raise pb.Error(value) + + reconnected = self._agent_id is not None + self._agent_id, self._ac_id = value + + if reconnected: + self.acread_data(force=True) + self.acwrite_data("", force=True) + return + + # start bridge + port = reactor.listenTCP(0, self._bridge_factory) + address = port.getHost() + self.log.debug("Serial Bridge is started on {address!r}", address=address) + if "sock" in self.cmd_options: + with open( + os.path.join(self.cmd_options["sock"], "sock"), + mode="w", + encoding="utf8", + ) as fp: + fp.write("socket://localhost:%d" % address.port) + + def client_terminal_stopped(self): + try: + d = self.agentpool.callRemote("acclose", self._agent_id, self._ac_id) + d.addCallback(lambda r: self.disconnect()) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def acread_data(self, force=False): + if force and self._d_acread: + self._d_acread.cancel() + self._d_acread = None + + if ( + self._d_acread and not self._d_acread.called + ) or not self._bridge_factory.has_clients(): + return + + try: + self._d_acread = self.agentpool.callRemote( + "acread", self._agent_id, self._ac_id + ) + self._d_acread.addCallback(self.cb_acread_result) + self._d_acread.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_acread_result(self, result): + if result is None: + self.disconnect(exit_code=1) + else: + self._bridge_factory.send_to_clients(result) + self.acread_data() + + def acwrite_data(self, data, force=False): + if force and self._d_acwrite: + self._d_acwrite.cancel() + self._d_acwrite = None + + self._acwrite_buffer += data + if len(self._acwrite_buffer) > self.MAX_BUFFER_SIZE: + self._acwrite_buffer = self._acwrite_buffer[-1 * self.MAX_BUFFER_SIZE :] + if (self._d_acwrite and not self._d_acwrite.called) or not self._acwrite_buffer: + return + + data = self._acwrite_buffer + self._acwrite_buffer = b"" + try: + d = self.agentpool.callRemote("acwrite", self._agent_id, self._ac_id, data) + d.addCallback(self.cb_acwrite_result) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_acwrite_result(self, result): + assert result > 0 + if self._acwrite_buffer: + self.acwrite_data(b"") diff --git a/platformio/remote/client/run_or_test.py b/platformio/remote/client/run_or_test.py new file mode 100644 index 0000000..d23d92c --- /dev/null +++ b/platformio/remote/client/run_or_test.py @@ -0,0 +1,271 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +import os +import zlib +from io import BytesIO + +from twisted.spread import pb # pylint: disable=import-error + +from platformio import fs +from platformio.compat import hashlib_encode_data +from platformio.project.config import ProjectConfig +from platformio.remote.client.async_base import AsyncClientBase +from platformio.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync + + +class RunOrTestClient(AsyncClientBase): + MAX_ARCHIVE_SIZE = 50 * 1024 * 1024 # 50Mb + UPLOAD_CHUNK_SIZE = 256 * 1024 # 256Kb + + PSYNC_SRC_EXTS = [ + "c", + "cpp", + "S", + "spp", + "SPP", + "sx", + "s", + "asm", + "ASM", + "h", + "hpp", + "ipp", + "ino", + "pde", + "json", + "properties", + ] + + PSYNC_SKIP_DIRS = (".git", ".svn", ".hg", "example", "examples", "test", "tests") + + def __init__(self, *args, **kwargs): + AsyncClientBase.__init__(self, *args, **kwargs) + self.project_id = self.generate_project_id(self.options["project_dir"]) + self.psync = ProjectSync(self.options["project_dir"]) + + def generate_project_id(self, path): + h = hashlib.sha1(hashlib_encode_data(self.id)) + h.update(hashlib_encode_data(path)) + return "%s-%s" % (os.path.basename(path), h.hexdigest()) + + def add_project_items(self, psync): + with fs.cd(self.options["project_dir"]): + cfg = ProjectConfig.get_instance( + os.path.join(self.options["project_dir"], "platformio.ini") + ) + psync.add_item(cfg.path, "platformio.ini") + psync.add_item(cfg.get("platformio", "shared_dir"), "shared") + psync.add_item(cfg.get("platformio", "boards_dir"), "boards") + + if self.options["force_remote"]: + self._add_project_source_items(cfg, psync) + else: + self._add_project_binary_items(cfg, psync) + + if self.command == "test": + psync.add_item(cfg.get("platformio", "test_dir"), "test") + + def _add_project_source_items(self, cfg, psync): + psync.add_item(cfg.get("platformio", "lib_dir"), "lib") + psync.add_item( + cfg.get("platformio", "include_dir"), + "include", + cb_filter=self._cb_tarfile_filter, + ) + psync.add_item( + cfg.get("platformio", "src_dir"), "src", cb_filter=self._cb_tarfile_filter + ) + if set(["buildfs", "uploadfs", "uploadfsota"]) & set( + self.options.get("target", []) + ): + psync.add_item(cfg.get("platformio", "data_dir"), "data") + + @staticmethod + def _add_project_binary_items(cfg, psync): + build_dir = cfg.get("platformio", "build_dir") + for env_name in os.listdir(build_dir): + env_dir = os.path.join(build_dir, env_name) + if not os.path.isdir(env_dir): + continue + for fname in os.listdir(env_dir): + bin_file = os.path.join(env_dir, fname) + bin_exts = (".elf", ".bin", ".hex", ".eep", "program") + if os.path.isfile(bin_file) and fname.endswith(bin_exts): + psync.add_item( + bin_file, os.path.join(".pio", "build", env_name, fname) + ) + + def _cb_tarfile_filter(self, path): + if ( + os.path.isdir(path) + and os.path.basename(path).lower() in self.PSYNC_SKIP_DIRS + ): + return None + if os.path.isfile(path) and not self.is_file_with_exts( + path, self.PSYNC_SRC_EXTS + ): + return None + return path + + @staticmethod + def is_file_with_exts(path, exts): + if path.endswith(tuple(".%s" % e for e in exts)): + return True + return False + + def agent_pool_ready(self): + self.psync_init() + + def psync_init(self): + self.add_project_items(self.psync) + d = self.agentpool.callRemote( + "cmd", + self.agents, + "psync", + dict(id=self.project_id, items=[i[1] for i in self.psync.get_items()]), + ) + d.addCallback(self.cb_psync_init_result) + d.addErrback(self.cb_global_error) + + # build db index while wait for result from agent + self.psync.rebuild_dbindex() + + def cb_psync_init_result(self, result): + self._acs_total = len(result) + for success, value in result: + if not success: + raise pb.Error(value) + agent_id, ac_id = value + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict(stage=PROJECT_SYNC_STAGE.DBINDEX.value), + ) + d.addCallback(self.cb_psync_dbindex_result, agent_id, ac_id) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_dbindex_result(self, result, agent_id, ac_id): + result = set(json.loads(zlib.decompress(result))) + dbindex = set(self.psync.get_dbindex()) + delete = list(result - dbindex) + delta = list(dbindex - result) + + self.log.debug( + "PSync: stats, total={total}, delete={delete}, delta={delta}", + total=len(dbindex), + delete=len(delete), + delta=len(delta), + ) + + if not delete and not delta: + return self.psync_finalize(agent_id, ac_id) + if not delete: + return self.psync_upload(agent_id, ac_id, delta) + + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict( + stage=PROJECT_SYNC_STAGE.DELETE.value, + dbindex=zlib.compress(json.dumps(delete).encode()), + ), + ) + d.addCallback(self.cb_psync_delete_result, agent_id, ac_id, delta) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + return None + + def cb_psync_delete_result(self, result, agent_id, ac_id, dbindex): + assert result + self.psync_upload(agent_id, ac_id, dbindex) + + def psync_upload(self, agent_id, ac_id, dbindex): + assert dbindex + fileobj = BytesIO() + compressed = self.psync.compress_items(fileobj, dbindex, self.MAX_ARCHIVE_SIZE) + fileobj.seek(0) + self.log.debug( + "PSync: upload project, size={size}", size=len(fileobj.getvalue()) + ) + self.psync_upload_chunk( + agent_id, ac_id, list(set(dbindex) - set(compressed)), fileobj + ) + + def psync_upload_chunk(self, agent_id, ac_id, dbindex, fileobj): + offset = fileobj.tell() + total = fileobj.seek(0, os.SEEK_END) + # unwind + fileobj.seek(offset) + chunk = fileobj.read(self.UPLOAD_CHUNK_SIZE) + assert chunk + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict( + stage=PROJECT_SYNC_STAGE.UPLOAD.value, + chunk=chunk, + length=len(chunk), + total=total, + ), + ) + d.addCallback( + self.cb_psync_upload_chunk_result, agent_id, ac_id, dbindex, fileobj + ) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_upload_chunk_result( # pylint: disable=too-many-arguments + self, result, agent_id, ac_id, dbindex, fileobj + ): + result = PROJECT_SYNC_STAGE.lookupByValue(result) + self.log.debug("PSync: upload chunk result {r}", r=str(result)) + assert result & (PROJECT_SYNC_STAGE.UPLOAD | PROJECT_SYNC_STAGE.EXTRACTED) + if result is PROJECT_SYNC_STAGE.EXTRACTED: + if dbindex: + self.psync_upload(agent_id, ac_id, dbindex) + else: + self.psync_finalize(agent_id, ac_id) + else: + self.psync_upload_chunk(agent_id, ac_id, dbindex, fileobj) + + def psync_finalize(self, agent_id, ac_id): + try: + d = self.agentpool.callRemote("acclose", agent_id, ac_id) + d.addCallback(self.cb_psync_completed_result, agent_id) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_completed_result(self, result, agent_id): + assert PROJECT_SYNC_STAGE.lookupByValue(result) + options = self.options.copy() + del options["project_dir"] + options["project_id"] = self.project_id + d = self.agentpool.callRemote("cmd", [agent_id], self.command, options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) diff --git a/platformio/remote/client/update_core.py b/platformio/remote/client/update_core.py new file mode 100644 index 0000000..4fdec4c --- /dev/null +++ b/platformio/remote/client/update_core.py @@ -0,0 +1,22 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.remote.client.async_base import AsyncClientBase + + +class UpdateCoreClient(AsyncClientBase): + def agent_pool_ready(self): + d = self.agentpool.callRemote("cmd", self.agents, self.command, self.options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) diff --git a/platformio/remote/factory/__init__.py b/platformio/remote/factory/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/remote/factory/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/remote/factory/client.py b/platformio/remote/factory/client.py new file mode 100644 index 0000000..b9e3cd5 --- /dev/null +++ b/platformio/remote/factory/client.py @@ -0,0 +1,86 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.cred import credentials # pylint: disable=import-error +from twisted.internet import defer, protocol, reactor # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio.account.client import AccountClient +from platformio.app import get_host_id + + +class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory): + def clientConnectionMade(self, broker): + if self.sslContextFactory and not self.sslContextFactory.certificate_verified: + self.remote_client.log.error( + "A remote cloud could not prove that its security certificate is " + "from {host}. This may cause a misconfiguration or an attacker " + "intercepting your connection.", + host=self.sslContextFactory.host, + ) + return self.remote_client.disconnect() + pb.PBClientFactory.clientConnectionMade(self, broker) + protocol.ReconnectingClientFactory.resetDelay(self) + self.remote_client.log.info("Successfully connected") + self.remote_client.log.info("Authenticating") + + auth_token = None + try: + auth_token = AccountClient().fetch_authentication_token() + except Exception as exc: # pylint:disable=broad-except + d = defer.Deferred() + d.addErrback(self.clientAuthorizationFailed) + d.errback(pb.Error(exc)) + return d + + d = self.login( + credentials.UsernamePassword( + auth_token.encode(), + get_host_id().encode(), + ), + client=self.remote_client, + ) + d.addCallback(self.remote_client.cb_client_authorization_made) + d.addErrback(self.clientAuthorizationFailed) + return d + + def clientAuthorizationFailed(self, err): + AccountClient.delete_local_session() + self.remote_client.cb_client_authorization_failed(err) + + def clientConnectionFailed(self, connector, reason): + self.remote_client.log.warn( + "Could not connect to PIO Remote Cloud. Reconnecting..." + ) + self.remote_client.cb_disconnected(reason) + protocol.ReconnectingClientFactory.clientConnectionFailed( + self, connector, reason + ) + + def clientConnectionLost( # pylint: disable=arguments-differ + self, connector, unused_reason + ): + if not reactor.running: + self.remote_client.log.info("Successfully disconnected") + return + self.remote_client.log.warn( + "Connection is lost to PIO Remote Cloud. Reconnecting" + ) + pb.PBClientFactory.clientConnectionLost( + self, connector, unused_reason, reconnecting=1 + ) + self.remote_client.cb_disconnected(unused_reason) + protocol.ReconnectingClientFactory.clientConnectionLost( + self, connector, unused_reason + ) diff --git a/platformio/remote/factory/ssl.py b/platformio/remote/factory/ssl.py new file mode 100644 index 0000000..78aa5d7 --- /dev/null +++ b/platformio/remote/factory/ssl.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import certifi +from OpenSSL import SSL # pylint: disable=import-error +from twisted.internet import ssl # pylint: disable=import-error + + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, host): + self.host = host + self.certificate_verified = False + + def getContext(self): + ctx = super().getContext() + ctx.set_verify( + SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname + ) + ctx.load_verify_locations(certifi.where()) + return ctx + + def verifyHostname( # pylint: disable=unused-argument,too-many-arguments + self, connection, x509, errno, depth, status + ): + cn = x509.get_subject().commonName + if cn.startswith("*"): + cn = cn[1:] + if self.host.endswith(cn): + self.certificate_verified = True + return status diff --git a/platformio/remote/projectsync.py b/platformio/remote/projectsync.py new file mode 100644 index 0000000..1715145 --- /dev/null +++ b/platformio/remote/projectsync.py @@ -0,0 +1,117 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tarfile +from binascii import crc32 +from os.path import getmtime, getsize, isdir, isfile, join + +from twisted.python import constants # pylint: disable=import-error + +from platformio.compat import hashlib_encode_data + + +class PROJECT_SYNC_STAGE(constants.Flags): + INIT = constants.FlagConstant() + DBINDEX = constants.FlagConstant() + DELETE = constants.FlagConstant() + UPLOAD = constants.FlagConstant() + EXTRACTED = constants.FlagConstant() + COMPLETED = constants.FlagConstant() + + +class ProjectSync: + def __init__(self, path): + self.path = path + if not isdir(self.path): + os.makedirs(self.path) + self.items = [] + self._db = {} + + def add_item(self, path, relpath, cb_filter=None): + self.items.append((path, relpath, cb_filter)) + + def get_items(self): + return self.items + + def rebuild_dbindex(self): + self._db = {} + for path, relpath, cb_filter in self.items: + if cb_filter and not cb_filter(path): + continue + self._insert_to_db(path, relpath) + if not isdir(path): + continue + for root, _, files in os.walk(path, followlinks=True): + for name in files: + self._insert_to_db( + join(root, name), join(relpath, root[len(path) + 1 :], name) + ) + + def _insert_to_db(self, path, relpath): + if not isfile(path): + return + index_hash = "%s-%s-%s" % (relpath, getmtime(path), getsize(path)) + index = crc32(hashlib_encode_data(index_hash)) + self._db[index] = (path, relpath) + + def get_dbindex(self): + return list(self._db.keys()) + + def delete_dbindex(self, dbindex): + for index in dbindex: + if index not in self._db: + continue + path = self._db[index][0] + if isfile(path): + os.remove(path) + del self._db[index] + self.delete_empty_folders() + return True + + def delete_empty_folders(self): + deleted = False + for item in self.items: + if not isdir(item[0]): + continue + for root, dirs, files in os.walk(item[0]): + if not dirs and not files and root != item[0]: + deleted = True + os.rmdir(root) + if deleted: + return self.delete_empty_folders() + + return True + + def compress_items(self, fileobj, dbindex, max_size): + compressed = [] + total_size = 0 + tar_opts = dict(fileobj=fileobj, mode="w:gz", bufsize=0, dereference=True) + with tarfile.open(**tar_opts) as tgz: + for index in dbindex: + compressed.append(index) + if index not in self._db: + continue + path, relpath = self._db[index] + tgz.add(path, relpath) + total_size += getsize(path) + if total_size > max_size: + break + return compressed + + def decompress_items(self, fileobj): + fileobj.seek(0) + with tarfile.open(fileobj=fileobj, mode="r:gz") as tgz: + tgz.extractall(self.path) + return True diff --git a/platformio/run/__init__.py b/platformio/run/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/run/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/run/cli.py b/platformio/run/cli.py new file mode 100644 index 0000000..1ed84a4 --- /dev/null +++ b/platformio/run/cli.py @@ -0,0 +1,336 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator +import os +import shutil +from multiprocessing import cpu_count +from time import time + +import click +from tabulate import tabulate + +from platformio import app, exception, fs, util +from platformio.device.monitor.command import device_monitor_cmd +from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectError +from platformio.project.helpers import find_project_dir_above, load_build_metadata +from platformio.run.helpers import clean_build_dir +from platformio.run.processor import EnvironmentProcessor +from platformio.test.runners.base import CTX_META_TEST_IS_RUNNING + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches + +try: + DEFAULT_JOB_NUMS = cpu_count() +except NotImplementedError: + DEFAULT_JOB_NUMS = 1 + + +@click.command("run", short_help="Run project targets (build, upload, clean, etc.)") +@click.option("-e", "--environment", multiple=True) +@click.option("-t", "--target", multiple=True) +@click.option("--upload-port") +@click.option("--monitor-port") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), +) +@click.option( + "-c", + "--project-conf", + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), +) +@click.option( + "-j", + "--jobs", + type=int, + default=DEFAULT_JOB_NUMS, + help=( + "Allow N jobs at once. " + "Default is a number of CPUs in a system (N=%d)" % DEFAULT_JOB_NUMS + ), +) +@click.option( + "-a", + "--program-arg", + "program_args", + multiple=True, + help="A program argument (multiple are allowed)", +) +@click.option("--disable-auto-clean", is_flag=True) +@click.option("--list-targets", is_flag=True) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.pass_context +def cli( + ctx, + environment, + target, + upload_port, + monitor_port, + project_dir, + project_conf, + jobs, + program_args, + disable_auto_clean, + list_targets, + silent, + verbose, +): + app.set_session_var("custom_project_conf", project_conf) + + # find project directory on upper level + if os.path.isfile(project_dir): + project_dir = find_project_dir_above(project_dir) + + targets = list(target) if target else [] + del target + only_monitor = targets == ["monitor"] + is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta + command_failed = False + + with fs.cd(project_dir): + config = ProjectConfig.get_instance(project_conf) + config.validate(environment) + + if list_targets: + return print_target_list(list(environment) or config.envs()) + + # clean obsolete build dir + if not only_monitor and not disable_auto_clean: + build_dir = config.get("platformio", "build_dir") + try: + clean_build_dir(build_dir, config) + except ProjectError as exc: + raise exc + except: # pylint: disable=bare-except + click.secho( + "Can not remove temporary directory `%s`. Please remove " + "it manually to avoid build issues" % build_dir, + fg="yellow", + ) + + default_envs = config.default_envs() + results = [] + for env in config.envs(): + skipenv = any( + [ + environment and env not in environment, + not environment and default_envs and env not in default_envs, + ] + ) + if skipenv: + results.append({"env": env}) + continue + + # print empty line between multi environment project + if not silent and any(r.get("succeeded") is not None for r in results): + click.echo() + + results.append( + process_env( + ctx, + env, + config, + targets, + upload_port, + monitor_port, + jobs, + program_args, + is_test_running, + silent, + verbose, + ) + ) + command_failed = any(r.get("succeeded") is False for r in results) + if ( + not is_test_running + and not only_monitor + and (command_failed or not silent) + and len(results) > 1 + ): + print_processing_summary(results, verbose) + + # Reset custom project config + app.set_session_var("custom_project_conf", None) + + if command_failed: + raise exception.ReturnErrorCode(1) + + return True + + +def process_env( + ctx, + name, + config, + targets, + upload_port, + monitor_port, + jobs, + program_args, + is_test_running, + silent, + verbose, +): + if not is_test_running and not silent: + print_processing_header(name, config, verbose) + + targets = targets or config.get(f"env:{name}", "targets", []) + only_monitor = targets == ["monitor"] + result = {"env": name, "duration": time(), "succeeded": True} + + if not only_monitor: + result["succeeded"] = EnvironmentProcessor( + ctx, + name, + config, + [t for t in targets if t != "monitor"], + upload_port, + jobs, + program_args, + silent, + verbose, + ).process() + + if result["succeeded"] and "monitor" in targets and "nobuild" not in targets: + ctx.invoke( + device_monitor_cmd, + port=monitor_port, + environment=name, + ) + + result["duration"] = time() - result["duration"] + + # print footer on error or when is not unit testing + if ( + not is_test_running + and not only_monitor + and (not silent or not result["succeeded"]) + ): + print_processing_footer(result) + + return result + + +def print_processing_header(env, config, verbose=False): + env_dump = [] + for k, v in config.items(env=env): + if verbose or k in ("platform", "framework", "board"): + env_dump.append("%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) + click.echo( + "Processing %s (%s)" + % (click.style(env, fg="cyan", bold=True), "; ".join(env_dump)) + ) + terminal_width = shutil.get_terminal_size().columns + click.secho("-" * terminal_width, bold=True) + + +def print_processing_footer(result): + is_failed = not result.get("succeeded") + util.print_labeled_bar( + "[%s] Took %.2f seconds" + % ( + ( + click.style("FAILED", fg="red", bold=True) + if is_failed + else click.style("SUCCESS", fg="green", bold=True) + ), + result["duration"], + ), + is_error=is_failed, + ) + + +def print_processing_summary(results, verbose=False): + tabular_data = [] + succeeded_nums = 0 + failed_nums = 0 + duration = 0 + + for result in results: + duration += result.get("duration", 0) + if result.get("succeeded") is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif result.get("succeeded") is None: + if not verbose: + continue + status_str = "IGNORED" + else: + succeeded_nums += 1 + status_str = click.style("SUCCESS", fg="green") + + tabular_data.append( + ( + click.style(result["env"], fg="cyan"), + status_str, + util.humanize_duration_time(result.get("duration")), + ) + ) + + click.echo() + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) for s in ("Environment", "Status", "Duration") + ], + ), + err=failed_nums, + ) + + util.print_labeled_bar( + "%s%d succeeded in %s" + % ( + "%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, + util.humanize_duration_time(duration), + ), + is_error=failed_nums, + fg="red" if failed_nums else "green", + ) + + +def print_target_list(envs): + tabular_data = [] + for env, data in load_build_metadata(os.getcwd(), envs).items(): + tabular_data.extend( + sorted( + [ + ( + click.style(env, fg="cyan"), + t["group"], + click.style(t.get("name"), fg="yellow"), + t["title"], + t.get("description"), + ) + for t in data.get("targets", []) + ], + key=operator.itemgetter(1, 2), + ) + ) + tabular_data.append((None, None, None, None, None)) + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Group", "Name", "Title", "Description") + ], + ), + ) diff --git a/platformio/run/helpers.py b/platformio/run/helpers.py new file mode 100644 index 0000000..f8bfdcb --- /dev/null +++ b/platformio/run/helpers.py @@ -0,0 +1,45 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import makedirs +from os.path import isdir, isfile, join + +from platformio import fs +from platformio.project.helpers import compute_project_checksum, get_project_dir + +KNOWN_CLEAN_TARGETS = ("clean",) +KNOWN_FULLCLEAN_TARGETS = ("cleanall", "fullclean") +KNOWN_ALLCLEAN_TARGETS = KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS + + +def clean_build_dir(build_dir, config): + # remove legacy ".pioenvs" folder + legacy_build_dir = join(get_project_dir(), ".pioenvs") + if isdir(legacy_build_dir) and legacy_build_dir != build_dir: + fs.rmtree(legacy_build_dir) + + checksum_file = join(build_dir, "project.checksum") + checksum = compute_project_checksum(config) + + if isdir(build_dir): + # check project structure + if isfile(checksum_file): + with open(checksum_file, encoding="utf8") as fp: + if fp.read() == checksum: + return + fs.rmtree(build_dir) + + makedirs(build_dir) + with open(checksum_file, mode="w", encoding="utf8") as fp: + fp.write(checksum) diff --git a/platformio/run/processor.py b/platformio/run/processor.py new file mode 100644 index 0000000..051e35a --- /dev/null +++ b/platformio/run/processor.py @@ -0,0 +1,91 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.commands.install import install_project_env_dependencies +from platformio.platform.factory import PlatformFactory +from platformio.project.exception import UndefinedEnvPlatformError +from platformio.run.helpers import KNOWN_ALLCLEAN_TARGETS +from platformio.test.runners.base import CTX_META_TEST_RUNNING_NAME + +# pylint: disable=too-many-instance-attributes + + +class EnvironmentProcessor: + def __init__( # pylint: disable=too-many-arguments + self, + cmd_ctx, + name, + config, + targets, + upload_port, + jobs, + program_args, + silent, + verbose, + ): + self.cmd_ctx = cmd_ctx + self.name = name + self.config = config + self.targets = targets + self.upload_port = upload_port + self.jobs = jobs + self.program_args = program_args + self.silent = silent + self.verbose = verbose + self.options = config.items(env=name, as_dict=True) + + def get_build_variables(self): + variables = dict( + pioenv=self.name, + project_config=self.config.path, + program_args=self.program_args, + ) + + if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta: + variables["piotest_running_name"] = self.cmd_ctx.meta[ + CTX_META_TEST_RUNNING_NAME + ] + + if self.upload_port: + # override upload port with a custom from CLI + variables["upload_port"] = self.upload_port + return variables + + def process(self): + if "platform" not in self.options: + raise UndefinedEnvPlatformError(self.name) + + build_vars = self.get_build_variables() + is_clean = set(KNOWN_ALLCLEAN_TARGETS) & set(self.targets) + build_targets = [t for t in self.targets if t not in KNOWN_ALLCLEAN_TARGETS] + + # pre-clean + if is_clean: + result = PlatformFactory.from_env( + self.name, targets=self.targets, autoinstall=True + ).run(build_vars, self.targets, self.silent, self.verbose, self.jobs) + if not build_targets: + return result["returncode"] == 0 + + install_project_env_dependencies( + self.name, + { + "project_targets": self.targets, + "piotest_running_name": build_vars.get("piotest_running_name"), + }, + ) + result = PlatformFactory.from_env( + self.name, targets=build_targets, autoinstall=True + ).run(build_vars, build_targets, self.silent, self.verbose, self.jobs) + return result["returncode"] == 0 diff --git a/platformio/system/__init__.py b/platformio/system/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/system/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/system/cli.py b/platformio/system/cli.py new file mode 100644 index 0000000..dd39ce8 --- /dev/null +++ b/platformio/system/cli.py @@ -0,0 +1,32 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.system.commands.completion import system_completion_cmd +from platformio.system.commands.info import system_info_cmd +from platformio.system.commands.prune import system_prune_cmd + + +@click.group( + "system", + commands=[ + system_completion_cmd, + system_info_cmd, + system_prune_cmd, + ], + short_help="Miscellaneous system commands", +) +def cli(): + pass diff --git a/platformio/system/commands/__init__.py b/platformio/system/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/system/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/system/commands/completion.py b/platformio/system/commands/completion.py new file mode 100644 index 0000000..5ff73bb --- /dev/null +++ b/platformio/system/commands/completion.py @@ -0,0 +1,69 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.system.completion import ( + ShellType, + get_completion_install_path, + install_completion_code, + uninstall_completion_code, +) + + +@click.group("completion", short_help="Shell completion support") +def system_completion_cmd(): + pass + + +@system_completion_cmd.command( + "install", short_help="Install shell completion files/code" +) +@click.argument("shell", type=click.Choice([t.value for t in ShellType])) +@click.option( + "--path", + type=click.Path(file_okay=True, dir_okay=False, readable=True), + help="Custom installation path of the code to be evaluated by the shell. " + "The standard installation path is used by default.", +) +def system_completion_install(shell, path): + shell = ShellType(shell) + path = path or get_completion_install_path(shell) + install_completion_code(shell, path) + click.echo( + "PlatformIO CLI completion has been installed for %s shell to %s \n" + "Please restart a current shell session." + % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue")) + ) + + +@system_completion_cmd.command( + "uninstall", short_help="Uninstall shell completion files/code" +) +@click.argument("shell", type=click.Choice([t.value for t in ShellType])) +@click.option( + "--path", + type=click.Path(file_okay=True, dir_okay=False, readable=True), + help="Custom installation path of the code to be evaluated by the shell. " + "The standard installation path is used by default.", +) +def system_completion_uninstall(shell, path): + shell = ShellType(shell) + path = path or get_completion_install_path(shell) + uninstall_completion_code(shell, path) + click.echo( + "PlatformIO CLI completion has been uninstalled for %s shell from %s \n" + "Please restart a current shell session." + % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue")) + ) diff --git a/platformio/system/commands/info.py b/platformio/system/commands/info.py new file mode 100644 index 0000000..4db1f42 --- /dev/null +++ b/platformio/system/commands/info.py @@ -0,0 +1,84 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import platform +import sys + +import click +from tabulate import tabulate + +from platformio import __version__, compat, proc, util +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.project.config import ProjectConfig + + +@click.command("info", short_help="Display system-wide information") +@click.option("--json-output", is_flag=True) +def system_info_cmd(json_output): + project_config = ProjectConfig() + data = {} + data["core_version"] = {"title": "PlatformIO Core", "value": __version__} + data["python_version"] = { + "title": "Python", + "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)), + } + data["system"] = {"title": "System Type", "value": util.get_systype()} + data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)} + data["filesystem_encoding"] = { + "title": "File System Encoding", + "value": compat.get_filesystem_encoding(), + } + data["locale_encoding"] = { + "title": "Locale Encoding", + "value": compat.get_locale_encoding(), + } + data["core_dir"] = { + "title": "PlatformIO Core Directory", + "value": project_config.get("platformio", "core_dir"), + } + data["platformio_exe"] = { + "title": "PlatformIO Core Executable", + "value": proc.where_is_program( + "platformio.exe" if compat.IS_WINDOWS else "platformio" + ), + } + data["python_exe"] = { + "title": "Python Executable", + "value": proc.get_pythonexe_path(), + } + data["global_lib_nums"] = { + "title": "Global Libraries", + "value": len(LibraryPackageManager().get_installed()), + } + data["dev_platform_nums"] = { + "title": "Development Platforms", + "value": len(PlatformPackageManager().get_installed()), + } + data["package_tool_nums"] = { + "title": "Tools & Toolchains", + "value": len( + ToolPackageManager( + project_config.get("platformio", "packages_dir") + ).get_installed() + ), + } + click.echo( + json.dumps(data) + if json_output + else tabulate([(item["title"], item["value"]) for item in data.values()]) + ) diff --git a/platformio/system/commands/prune.py b/platformio/system/commands/prune.py new file mode 100644 index 0000000..1559aab --- /dev/null +++ b/platformio/system/commands/prune.py @@ -0,0 +1,70 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio import fs +from platformio.system.prune import ( + prune_cached_data, + prune_core_packages, + prune_platform_packages, +) + + +@click.command("prune", short_help="Remove unused data") +@click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation") +@click.option( + "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed" +) +@click.option("--cache", is_flag=True, help="Prune only cached data") +@click.option( + "--core-packages", is_flag=True, help="Prune only unnecessary core packages" +) +@click.option( + "--platform-packages", + is_flag=True, + help="Prune only unnecessary development platform packages", +) +def system_prune_cmd(force, dry_run, cache, core_packages, platform_packages): + if dry_run: + click.secho( + "Dry run mode (do not prune, only show data that will be removed)", + fg="yellow", + ) + click.echo() + + reclaimed_cache = 0 + reclaimed_core_packages = 0 + reclaimed_platform_packages = 0 + prune_all = not any([cache, core_packages, platform_packages]) + + if cache or prune_all: + reclaimed_cache = prune_cached_data(force, dry_run) + click.echo() + + if core_packages or prune_all: + reclaimed_core_packages = prune_core_packages(force, dry_run) + click.echo() + + if platform_packages or prune_all: + reclaimed_platform_packages = prune_platform_packages(force, dry_run) + click.echo() + + click.secho( + "Total reclaimed space: %s" + % fs.humanize_file_size( + reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages + ), + fg="green", + ) diff --git a/platformio/system/completion.py b/platformio/system/completion.py new file mode 100644 index 0000000..100d1a1 --- /dev/null +++ b/platformio/system/completion.py @@ -0,0 +1,101 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import subprocess +from enum import Enum + +import click + +from platformio.compat import IS_MACOS + + +class ShellType(Enum): + FISH = "fish" + ZSH = "zsh" + BASH = "bash" + + +def get_bash_version(): + output = subprocess.run( + ["bash", "--version"], check=True, stdout=subprocess.PIPE + ).stdout.decode() + match = re.search(r"version\s+(\d+)\.(\d+)", output, re.IGNORECASE) + if match: + return (int(match.group(1)), int(match.group(2))) + return (0, 0) + + +def get_completion_install_path(shell): + home_dir = os.path.expanduser("~") + prog_name = click.get_current_context().find_root().info_name + if shell == ShellType.FISH: + return os.path.join( + home_dir, ".config", "fish", "completions", "%s.fish" % prog_name + ) + if shell == ShellType.ZSH: + return os.path.join(home_dir, ".zshrc") + if shell == ShellType.BASH: + return os.path.join(home_dir, ".bash_completion") + raise click.ClickException("%s is not supported." % shell) + + +def get_completion_code(shell): + if shell == ShellType.FISH: + return "eval (env _PIO_COMPLETE=fish_source pio)" + if shell == ShellType.ZSH: + code = "autoload -Uz compinit\ncompinit\n" if IS_MACOS else "" + return code + 'eval "$(_PIO_COMPLETE=zsh_source pio)"' + if shell == ShellType.BASH: + return 'eval "$(_PIO_COMPLETE=bash_source pio)"' + raise click.ClickException("%s is not supported." % shell) + + +def is_completion_code_installed(shell, path): + if shell == ShellType.FISH or not os.path.exists(path): + return False + with open(path, encoding="utf8") as fp: + return get_completion_code(shell) in fp.read() + + +def install_completion_code(shell, path): + if shell == ShellType.BASH and get_bash_version() < (4, 4): + raise click.ClickException("The minimal supported Bash version is 4.4") + if is_completion_code_installed(shell, path): + return None + append = shell != ShellType.FISH + with open(path, mode="a" if append else "w", encoding="utf8") as fp: + if append: + fp.write("\n\n# Begin: PlatformIO Core completion support\n") + fp.write(get_completion_code(shell)) + if append: + fp.write("\n# End: PlatformIO Core completion support\n\n") + return True + + +def uninstall_completion_code(shell, path): + if not os.path.exists(path): + return True + if shell == ShellType.FISH: + os.remove(path) + return True + + with open(path, "r+", encoding="utf8") as fp: + contents = fp.read() + fp.seek(0) + fp.truncate() + fp.write(contents.replace(get_completion_code(shell), "")) + + return True diff --git a/platformio/system/prune.py b/platformio/system/prune.py new file mode 100644 index 0000000..e00da69 --- /dev/null +++ b/platformio/system/prune.py @@ -0,0 +1,98 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from operator import itemgetter + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.package.manager.core import remove_unnecessary_core_packages +from platformio.package.manager.platform import remove_unnecessary_platform_packages +from platformio.project.helpers import get_project_cache_dir + + +def prune_cached_data(force=False, dry_run=False, silent=False): + reclaimed_space = 0 + if not silent: + click.secho("Prune cached data:", bold=True) + click.echo(" - cached API requests") + click.echo(" - cached package downloads") + click.echo(" - temporary data") + cache_dir = get_project_cache_dir() + if os.path.isdir(cache_dir): + reclaimed_space += fs.calculate_folder_size(cache_dir) + if not dry_run: + if not force: + click.confirm("Do you want to continue?", abort=True) + fs.rmtree(cache_dir) + if not silent: + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + return reclaimed_space + + +def prune_core_packages(force=False, dry_run=False, silent=False): + if not silent: + click.secho("Prune unnecessary core packages:", bold=True) + return _prune_packages(force, dry_run, silent, remove_unnecessary_core_packages) + + +def prune_platform_packages(force=False, dry_run=False, silent=False): + if not silent: + click.secho("Prune unnecessary development platform packages:", bold=True) + return _prune_packages(force, dry_run, silent, remove_unnecessary_platform_packages) + + +def _prune_packages(force, dry_run, silent, handler): + if not silent: + click.echo("Calculating...") + items = [ + ( + pkg, + fs.calculate_folder_size(pkg.path), + ) + for pkg in handler(dry_run=True) + ] + items = sorted(items, key=itemgetter(1), reverse=True) + reclaimed_space = sum(item[1] for item in items) + if items and not silent: + click.echo( + tabulate( + [ + ( + pkg.metadata.spec.humanize(), + str(pkg.metadata.version), + fs.humanize_file_size(size), + ) + for (pkg, size) in items + ], + headers=["Package", "Version", "Size"], + ) + ) + if not dry_run: + if not force: + click.confirm("Do you want to continue?", abort=True) + handler(dry_run=False) + if not silent: + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + return reclaimed_space + + +def calculate_unnecessary_system_data(): + return ( + prune_cached_data(force=True, dry_run=True, silent=True) + + prune_core_packages(force=True, dry_run=True, silent=True) + + prune_platform_packages(force=True, dry_run=True, silent=True) + ) diff --git a/platformio/telemetry.py b/platformio/telemetry.py new file mode 100644 index 0000000..5b0734c --- /dev/null +++ b/platformio/telemetry.py @@ -0,0 +1,380 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +import os +import queue +import re +import sys +import threading +import time +import traceback +from collections import deque + +import requests + +from platformio import __title__, __version__, app, exception, fs, util +from platformio.cli import PlatformioCLI +from platformio.debug.config.base import DebugConfigBase +from platformio.http import HTTPSession +from platformio.proc import is_ci + +KEEP_MAX_REPORTS = 100 +SEND_MAX_EVENTS = 25 + + +class MeasurementProtocol: + def __init__(self, events=None): + self.client_id = app.get_cid() + self._events = events or [] + self._user_properties = {} + + self.set_user_property("systype", util.get_systype()) + created_at = app.get_state_item("created_at", None) + if created_at: + self.set_user_property("created_at", int(created_at)) + + @staticmethod + def event_to_dict(name, params, timestamp=None): + event = {"name": name, "params": params} + if timestamp is not None: + event["timestamp"] = timestamp + return event + + def set_user_property(self, name, value): + self._user_properties[name] = value + + def add_event(self, name, params): + self._events.append(self.event_to_dict(name, params)) + + def to_payload(self): + return { + "client_id": self.client_id, + "user_properties": self._user_properties, + "events": self._events, + } + + +@util.singleton +class TelemetryLogger: + def __init__(self): + self._events = deque() + + self._sender_thread = None + self._sender_queue = queue.Queue() + self._sender_terminated = False + + self._http_session = HTTPSession() + self._http_offline = False + + def close(self): + self._http_session.close() + + def log_event(self, name, params, timestamp=None, instant_sending=False): + if not app.get_setting("enable_telemetry") or app.get_session_var( + "pause_telemetry" + ): + return None + timestamp = timestamp or int(time.time()) + self._events.append( + MeasurementProtocol.event_to_dict(name, params, timestamp=timestamp) + ) + if self._http_offline: # if network is off-line + return False + if instant_sending: + self.send() + return True + + def send(self): + if not self._events or self._sender_terminated: + return + if not self._sender_thread: + self._sender_thread = threading.Thread( + target=self._sender_worker, daemon=True + ) + self._sender_thread.start() + while self._events: + events = [] + try: + while len(events) < SEND_MAX_EVENTS: + events.append(self._events.popleft()) + except IndexError: + pass + self._sender_queue.put(events) + + def _sender_worker(self): + while True: + if self._sender_terminated: + return + try: + events = self._sender_queue.get() + if not self._commit_events(events): + self._events.extend(events) + self._sender_queue.task_done() + except (queue.Empty, ValueError): + pass + + def _commit_events(self, events): + if self._http_offline: + return False + mp = MeasurementProtocol(events) + payload = mp.to_payload() + # print("_commit_payload", payload) + try: + r = self._http_session.post( + "https://collector.platformio.org/collect", + json=payload, + timeout=(2, 5), # connect, read + ) + r.raise_for_status() + return True + except requests.exceptions.HTTPError as exc: + # skip Bad Request + if exc.response.status_code >= 400 and exc.response.status_code < 500: + return True + except: # pylint: disable=bare-except + pass + self._http_offline = True + return False + + def terminate_sender(self): + self._sender_terminated = True + + def is_sending(self): + return self._sender_queue.unfinished_tasks + + def get_unsent_events(self): + result = list(self._events) + try: + while True: + result.extend(self._sender_queue.get_nowait()) + except queue.Empty: + pass + return result + + +def log_event(name, params, instant_sending=False): + TelemetryLogger().log_event(name, params, instant_sending=instant_sending) + + +def on_cmd_start(cmd_ctx): + process_postponed_logs() + log_command(cmd_ctx) + + +def on_exit(): + TelemetryLogger().send() + + +def log_command(ctx): + params = { + "path_args": PlatformioCLI.reveal_cmd_path_args(ctx), + } + if is_ci(): + params["ci_actor"] = resolve_ci_actor() or "Unknown" + log_event("cmd_run", params) + + +def resolve_ci_actor(): + known_cis = ( + "GITHUB_ACTIONS", + "TRAVIS", + "APPVEYOR", + "GITLAB_CI", + "CIRCLECI", + "SHIPPABLE", + "DRONE", + ) + for name in known_cis: + if os.getenv(name, "false").lower() == "true": + return name + return None + + +def dump_project_env_params(config, env, platform): + non_sensitive_data = [ + "platform", + "framework", + "board", + "upload_protocol", + "check_tool", + "debug_tool", + "test_framework", + ] + section = f"env:{env}" + params = { + option: config.get(section, option) + for option in non_sensitive_data + if config.has_option(section, option) + } + params["pid"] = app.get_project_id(os.path.dirname(config.path)) + params["platform_name"] = platform.name + params["platform_version"] = platform.version + return params + + +def log_platform_run(platform, project_config, project_env, targets=None): + params = dump_project_env_params(project_config, project_env, platform) + if targets: + params["targets"] = targets + log_event("platform_run", params, instant_sending=True) + + +def log_exception(exc): + skip_conditions = [ + isinstance(exc, cls) + for cls in ( + IOError, + exception.ReturnErrorCode, + exception.UserSideException, + ) + ] + skip_conditions.append(not isinstance(exc, Exception)) + if any(skip_conditions): + return + is_fatal = any( + [ + not isinstance(exc, exception.PlatformioException), + "Error" in exc.__class__.__name__, + ] + ) + + def _strip_module_path(match): + module_path = match.group(1).replace(fs.get_source_dir() + os.sep, "") + sp_folder_name = "site-packages" + sp_pos = module_path.find(sp_folder_name) + if sp_pos != -1: + module_path = module_path[sp_pos + len(sp_folder_name) + 1 :] + module_path = fs.to_unix_path(module_path) + return f'File "{module_path}",' + + trace = re.sub( + r'File "([^"]+)",', + _strip_module_path, + traceback.format_exc(), + flags=re.MULTILINE, + ) + + params = { + "name": exc.__class__.__name__, + "description": str(exc), + "traceback": trace, + "cmd_args": sys.argv[1:], + "is_fatal": is_fatal, + } + log_event("exception", params) + + +def log_debug_started(debug_config: DebugConfigBase): + log_event( + "debug_started", + dump_project_env_params( + debug_config.project_config, debug_config.env_name, debug_config.platform + ), + ) + + +def log_debug_exception(exc, debug_config: DebugConfigBase): + # cleanup sensitive information, such as paths + description = fs.to_unix_path(str(exc)) + description = re.sub( + r'(^|\s+|")(?:[a-z]\:)?((/[^"/]+)+)(\s+|"|$)', + lambda m: " %s " % os.path.join(*m.group(2).split("/")[-2:]), + description, + re.I | re.M, + ) + params = { + "name": exc.__class__.__name__, + "description": description.strip(), + } + params.update( + dump_project_env_params( + debug_config.project_config, debug_config.env_name, debug_config.platform + ) + ) + log_event("debug_exception", params) + + +@atexit.register +def _finalize(): + timeout = 1000 # msec + elapsed = 0 + telemetry = TelemetryLogger() + telemetry.terminate_sender() + try: + while elapsed < timeout: + if not telemetry.is_sending(): + break + time.sleep(0.2) + elapsed += 200 + except KeyboardInterrupt: + pass + postpone_events(telemetry.get_unsent_events()) + telemetry.close() + + +def load_postponed_events(): + state_path = app.resolve_state_path( + "cache_dir", "telemetry.json", ensure_dir_exists=False + ) + if not os.path.isfile(state_path): + return [] + with app.State(state_path) as state: + return state.get("events", []) + + +def save_postponed_events(events): + state_path = app.resolve_state_path("cache_dir", "telemetry.json") + if not events: + try: + if os.path.isfile(state_path): + os.remove(state_path) + except: # pylint: disable=bare-except + pass + return None + with app.State(state_path, lock=True) as state: + state["events"] = events + state.modified = True + return True + + +def postpone_events(events): + if not events: + return None + postponed_events = load_postponed_events() or [] + timestamp = int(time.time()) + for event in events: + if "timestamp" not in event: + event["timestamp"] = timestamp + postponed_events.append(event) + save_postponed_events(postponed_events[KEEP_MAX_REPORTS * -1 :]) + return True + + +def process_postponed_logs(): + events = load_postponed_events() + if not events: + return None + save_postponed_events([]) # clean + telemetry = TelemetryLogger() + for event in events: + if set(["name", "params", "timestamp"]) <= set(event.keys()): + telemetry.log_event( + event["name"], + event["params"], + timestamp=event["timestamp"], + instant_sending=False, + ) + telemetry.send() + return True diff --git a/platformio/test/__init__.py b/platformio/test/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/test/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/test/cli.py b/platformio/test/cli.py new file mode 100644 index 0000000..748580c --- /dev/null +++ b/platformio/test/cli.py @@ -0,0 +1,203 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil + +import click + +from platformio import app, exception, fs, util +from platformio.project.config import ProjectConfig +from platformio.test.helpers import list_test_suites +from platformio.test.reports.base import TestReportFactory +from platformio.test.result import TestResult, TestStatus +from platformio.test.runners.base import TestRunnerOptions +from platformio.test.runners.factory import TestRunnerFactory + + +@click.command("test", short_help="Unit Testing") +@click.option("--environment", "-e", multiple=True) +@click.option( + "--filter", + "-f", + multiple=True, + metavar="PATTERN", + help="Filter tests by a pattern", +) +@click.option( + "--ignore", + "-i", + multiple=True, + metavar="PATTERN", + help="Ignore tests by a pattern", +) +@click.option("--upload-port") +@click.option("--test-port") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), +) +@click.option( + "-c", + "--project-conf", + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), +) +@click.option("--without-building", is_flag=True) +@click.option("--without-uploading", is_flag=True) +@click.option("--without-testing", is_flag=True) +@click.option("--no-reset", is_flag=True) +@click.option( + "--monitor-rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state for Serial Monitor", +) +@click.option( + "--monitor-dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state for Serial Monitor", +) +@click.option( + "-a", + "--program-arg", + "program_args", + multiple=True, + help="A program argument (multiple are allowed)", +) +@click.option("--list-tests", is_flag=True) +@click.option("--json-output-path", type=click.Path()) +@click.option("--junit-output-path", type=click.Path()) +@click.option( + "--verbose", + "-v", + count=True, + help="Increase verbosity level, maximum is 3 levels (-vvv), see docs for details", +) +@click.pass_context +def cli( # pylint: disable=too-many-arguments,too-many-locals,redefined-builtin + ctx, + environment, + ignore, + filter, + upload_port, + test_port, + project_dir, + project_conf, + without_building, + without_uploading, + without_testing, + no_reset, + monitor_rts, + monitor_dtr, + program_args, + list_tests, + json_output_path, + junit_output_path, + verbose, +): + app.set_session_var("custom_project_conf", project_conf) + + with fs.cd(project_dir): + project_config = ProjectConfig.get_instance(project_conf) + project_config.validate(envs=environment) + + test_result = TestResult(project_dir) + test_suites = list_test_suites( + project_config, environments=environment, filters=filter, ignores=ignore + ) + test_names = sorted(set(s.test_name for s in test_suites)) + + if not verbose: + click.echo("Verbosity level can be increased via `-v, -vv, or -vvv` option") + click.secho("Collected %d tests" % len(test_names), bold=True, nl=not verbose) + if verbose: + click.echo(" (%s)" % ", ".join(test_names)) + + for test_suite in test_suites: + test_result.add_suite(test_suite) + if list_tests or test_suite.is_finished(): # skipped by user + continue + runner = TestRunnerFactory.new( + test_suite, + project_config, + TestRunnerOptions( + verbose=verbose, + without_building=without_building, + without_uploading=without_uploading, + without_testing=without_testing, + upload_port=upload_port, + test_port=test_port, + no_reset=no_reset, + monitor_rts=monitor_rts, + monitor_dtr=monitor_dtr, + program_args=program_args, + ), + ) + click.echo() + print_suite_header(test_suite) + runner.start(ctx) + print_suite_footer(test_suite) + + stdout_report = TestReportFactory.new("stdout", test_result) + stdout_report.generate(verbose=verbose or list_tests) + + for output_format, output_path in [ + ("json", json_output_path), + ("junit", junit_output_path), + ]: + if not output_path: + continue + custom_report = TestReportFactory.new(output_format, test_result) + custom_report.generate(output_path=output_path, verbose=True) + + # Reset custom project config + app.set_session_var("custom_project_conf", None) + + if test_result.is_errored or test_result.get_status_nums(TestStatus.FAILED): + raise exception.ReturnErrorCode(1) + + +def print_suite_header(test_suite): + click.echo( + "Processing %s in %s environment" + % ( + click.style(test_suite.test_name, fg="yellow", bold=True), + click.style(test_suite.env_name, fg="cyan", bold=True), + ) + ) + terminal_width = shutil.get_terminal_size().columns + click.secho("-" * terminal_width, bold=True) + + +def print_suite_footer(test_suite): + is_error = test_suite.status in (TestStatus.FAILED, TestStatus.ERRORED) + util.print_labeled_bar( + "%s [%s] Took %.2f seconds" + % ( + click.style( + "%s:%s" % (test_suite.env_name, test_suite.test_name), bold=True + ), + ( + click.style(test_suite.status.name, fg="red", bold=True) + if is_error + else click.style("PASSED", fg="green", bold=True) + ), + test_suite.duration, + ), + is_error=is_error, + sep="-", + ) diff --git a/platformio/test/exception.py b/platformio/test/exception.py new file mode 100644 index 0000000..14cbe5a --- /dev/null +++ b/platformio/test/exception.py @@ -0,0 +1,33 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException, UserSideException + + +class UnitTestError(PlatformioException): + pass + + +class TestDirNotExistsError(UnitTestError, UserSideException): + MESSAGE = ( + "A test folder '{0}' does not exist.\nPlease create 'test' " + "directory in the project root and put a test suite.\n" + "More details about Unit " + "Testing: https://docs.platformio.org/en/latest/advanced/" + "unit-testing/index.html" + ) + + +class UnitTestSuiteError(UnitTestError): + pass diff --git a/platformio/test/helpers.py b/platformio/test/helpers.py new file mode 100644 index 0000000..25551ef --- /dev/null +++ b/platformio/test/helpers.py @@ -0,0 +1,73 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from fnmatch import fnmatch + +from platformio.test.exception import TestDirNotExistsError +from platformio.test.result import TestSuite + + +def list_test_names(project_config): + test_dir = project_config.get("platformio", "test_dir") + if not os.path.isdir(test_dir): + raise TestDirNotExistsError(test_dir) + names = [] + for root, _, __ in os.walk(test_dir, followlinks=True): + if not os.path.basename(root).startswith("test_"): + continue + names.append(os.path.relpath(root, test_dir).replace("\\", "/")) + if not names: + names = ["*"] + return names + + +def list_test_suites(project_config, environments, filters, ignores): + result = [] + test_dir = project_config.get("platformio", "test_dir") + default_envs = project_config.default_envs() + test_names = list_test_names(project_config) + for env_name in project_config.envs(): + for test_name in test_names: + # filter and ignore patterns + patterns = dict(filter=list(filters), ignore=list(ignores)) + for key, value in patterns.items(): + if value: # overridden from CLI + continue + patterns[key].extend( # pylint: disable=unnecessary-dict-index-lookup + project_config.get(f"env:{env_name}", f"test_{key}", []) + ) + + skip_conditions = [ + environments and env_name not in environments, + not environments and default_envs and env_name not in default_envs, + test_name != "*" + and patterns["filter"] + and not any(fnmatch(test_name, p) for p in patterns["filter"]), + test_name != "*" + and any(fnmatch(test_name, p) for p in patterns["ignore"]), + ] + result.append( + TestSuite( + env_name, + test_name, + finished=any(skip_conditions), + test_dir=os.path.abspath( + test_dir + if test_name == "*" + else os.path.join(test_dir, test_name) + ), + ) + ) + return result diff --git a/platformio/test/reports/__init__.py b/platformio/test/reports/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/test/reports/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/test/reports/base.py b/platformio/test/reports/base.py new file mode 100644 index 0000000..78244ae --- /dev/null +++ b/platformio/test/reports/base.py @@ -0,0 +1,34 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib + +from platformio.test.result import TestResult + + +class TestReportBase: + def __init__(self, test_result): + self.test_result = test_result + + def generate(self, output_path, verbose): + raise NotImplementedError() + + +class TestReportFactory: + @staticmethod + def new(format, test_result) -> TestReportBase: # pylint: disable=redefined-builtin + assert isinstance(test_result, TestResult) + mod = importlib.import_module(f"platformio.test.reports.{format}") + report_cls = getattr(mod, "%sTestReport" % format.lower().capitalize()) + return report_cls(test_result) diff --git a/platformio/test/reports/json.py b/platformio/test/reports/json.py new file mode 100644 index 0000000..791224c --- /dev/null +++ b/platformio/test/reports/json.py @@ -0,0 +1,100 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import json +import os + +import click + +from platformio.test.reports.base import TestReportBase +from platformio.test.result import TestStatus + + +class JsonTestReport(TestReportBase): + def generate(self, output_path, verbose=False): + if os.path.isdir(output_path): + output_path = os.path.join( + output_path, + "pio-test-report-%s-%s.json" + % ( + os.path.basename(self.test_result.project_dir), + datetime.datetime.now().strftime("%Y%m%d%H%M%S"), + ), + ) + + with open(output_path, mode="w", encoding="utf8") as fp: + json.dump(self.to_json(), fp) + + if verbose: + click.secho(f"Saved JSON report to the {output_path}", fg="green") + + def to_json(self): + result = dict( + version="1.0", + project_dir=self.test_result.project_dir, + duration=self.test_result.duration, + testcase_nums=self.test_result.case_nums, + error_nums=self.test_result.get_status_nums(TestStatus.ERRORED), + failure_nums=self.test_result.get_status_nums(TestStatus.FAILED), + skipped_nums=self.test_result.get_status_nums(TestStatus.SKIPPED), + test_suites=[], + ) + for test_suite in self.test_result.suites: + result["test_suites"].append(self.test_suite_to_json(test_suite)) + return result + + def test_suite_to_json(self, test_suite): + result = dict( + env_name=test_suite.env_name, + test_name=test_suite.test_name, + test_dir=test_suite.test_dir, + status=test_suite.status.name, + duration=test_suite.duration, + timestamp=datetime.datetime.fromtimestamp(test_suite.timestamp).strftime( + "%Y-%m-%dT%H:%M:%S" + ) + if test_suite.timestamp + else None, + testcase_nums=len(test_suite.cases), + error_nums=test_suite.get_status_nums(TestStatus.ERRORED), + failure_nums=test_suite.get_status_nums(TestStatus.FAILED), + skipped_nums=test_suite.get_status_nums(TestStatus.SKIPPED), + test_cases=[], + ) + for test_case in test_suite.cases: + result["test_cases"].append(self.test_case_to_json(test_case)) + return result + + @staticmethod + def test_case_to_json(test_case): + result = dict( + name=test_case.name, + status=test_case.status.name, + message=test_case.message, + stdout=test_case.stdout, + duration=test_case.duration, + exception=None, + source=None, + ) + if test_case.exception: + result["exception"] = "%s: %s" % ( + test_case.exception.__class__.__name__, + test_case.exception, + ) + if test_case.source: + result["source"] = dict( + file=test_case.source.filename, line=test_case.source.line + ) + return result diff --git a/platformio/test/reports/junit.py b/platformio/test/reports/junit.py new file mode 100644 index 0000000..34e177f --- /dev/null +++ b/platformio/test/reports/junit.py @@ -0,0 +1,107 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import xml.etree.ElementTree as ET + +import click + +from platformio import __version__ +from platformio.test.reports.base import TestReportBase +from platformio.test.result import TestStatus + + +class JunitTestReport(TestReportBase): + def generate(self, output_path, verbose=False): + if os.path.isdir(output_path): + output_path = os.path.join( + output_path, + "pio-test-report-%s-%s-junit.xml" + % ( + os.path.basename(self.test_result.project_dir), + datetime.datetime.now().strftime("%Y%m%d%H%M%S"), + ), + ) + + with open(output_path, mode="wb") as fp: + self.build_xml_tree().write(fp, encoding="utf8") + + if verbose: + click.secho(f"Saved JUnit report to the {output_path}", fg="green") + + def build_xml_tree(self): + root = ET.Element("testsuites") + root.set("name", self.test_result.project_dir) + root.set("platformio_version", __version__) + root.set("tests", str(self.test_result.case_nums)) + root.set("errors", str(self.test_result.get_status_nums(TestStatus.ERRORED))) + root.set("failures", str(self.test_result.get_status_nums(TestStatus.FAILED))) + root.set("time", str(self.test_result.duration)) + for suite in self.test_result.suites: + root.append(self.build_testsuite_node(suite)) + return ET.ElementTree(root) + + def build_testsuite_node(self, test_suite): + element = ET.Element("testsuite") + element.set("name", f"{test_suite.env_name}:{test_suite.test_name}") + element.set("tests", str(len(test_suite.cases))) + element.set("errors", str(test_suite.get_status_nums(TestStatus.ERRORED))) + element.set("failures", str(test_suite.get_status_nums(TestStatus.FAILED))) + element.set("skipped", str(test_suite.get_status_nums(TestStatus.SKIPPED))) + element.set("time", str(test_suite.duration)) + if test_suite.timestamp: + element.set( + "timestamp", + datetime.datetime.fromtimestamp(test_suite.timestamp).strftime( + "%Y-%m-%dT%H:%M:%S" + ), + ) + for test_case in test_suite.cases: + element.append(self.build_testcase_node(test_case)) + return element + + def build_testcase_node(self, test_case): + element = ET.Element("testcase") + element.set("name", str(test_case.name)) + element.set("time", str(test_case.duration)) + element.set("status", str(test_case.status.name)) + if test_case.source: + element.set("file", test_case.source.filename) + element.set("line", str(test_case.source.line)) + if test_case.status == TestStatus.SKIPPED: + element.append(ET.Element("skipped")) + elif test_case.status == TestStatus.ERRORED: + element.append(self.build_testcase_error_node(test_case)) + elif test_case.status == TestStatus.FAILED: + element.append(self.build_testcase_failure_node(test_case)) + return element + + @staticmethod + def build_testcase_error_node(test_case): + element = ET.Element("error") + element.set("type", test_case.exception.__class__.__name__) + element.set("message", str(test_case.exception)) + if test_case.stdout: + element.text = test_case.stdout + return element + + @staticmethod + def build_testcase_failure_node(test_case): + element = ET.Element("failure") + if test_case.message: + element.set("message", test_case.message) + if test_case.stdout: + element.text = test_case.stdout + return element diff --git a/platformio/test/reports/stdout.py b/platformio/test/reports/stdout.py new file mode 100644 index 0000000..2f37bf4 --- /dev/null +++ b/platformio/test/reports/stdout.py @@ -0,0 +1,99 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +from tabulate import tabulate + +from platformio import util +from platformio.test.reports.base import TestReportBase +from platformio.test.result import TestStatus + + +class StdoutTestReport(TestReportBase): + def generate(self, verbose=False): # pylint: disable=arguments-differ + click.echo() + + tabular_data = [] + failed_nums = self.test_result.get_status_nums(TestStatus.FAILED) + skipped_nums = self.test_result.get_status_nums(TestStatus.SKIPPED) + is_error = failed_nums > 0 or self.test_result.is_errored + + for test_suite in self.test_result.suites: + if not verbose and test_suite.status == TestStatus.SKIPPED: + continue + status_str = test_suite.status.name + if test_suite.status in (TestStatus.FAILED, TestStatus.ERRORED): + status_str = click.style(status_str, fg="red") + elif test_suite.status == TestStatus.PASSED: + status_str = click.style(status_str, fg="green") + + tabular_data.append( + ( + click.style(test_suite.env_name, fg="cyan"), + test_suite.test_name, + status_str, + util.humanize_duration_time(test_suite.duration or None), + ) + ) + + if tabular_data: + util.print_labeled_bar( + "SUMMARY", + is_error=is_error, + fg="red" if is_error else "green", + ) + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Test", "Status", "Duration") + ], + ), + err=is_error, + ) + + if failed_nums: + self.print_failed_test_cases() + + util.print_labeled_bar( + "%d test cases: %s%s%d succeeded in %s" + % ( + self.test_result.case_nums, + ("%d failed, " % failed_nums) if failed_nums else "", + ("%d skipped, " % skipped_nums) if skipped_nums else "", + self.test_result.get_status_nums(TestStatus.PASSED), + util.humanize_duration_time(self.test_result.duration), + ), + is_error=is_error, + fg="red" if is_error else "green", + ) + + def print_failed_test_cases(self): + click.echo() + for test_suite in self.test_result.suites: + if test_suite.status != TestStatus.FAILED: + continue + util.print_labeled_bar( + click.style( + f"{test_suite.env_name}:{test_suite.test_name}", bold=True, fg="red" + ), + is_error=True, + sep="_", + ) + for test_case in test_suite.cases: + if test_case.status != TestStatus.FAILED: + continue + click.echo((test_case.stdout or "").strip()) + click.echo() diff --git a/platformio/test/result.py b/platformio/test/result.py new file mode 100644 index 0000000..091fa96 --- /dev/null +++ b/platformio/test/result.py @@ -0,0 +1,166 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum +import functools +import operator +import time + +import click + + +class TestStatus(enum.Enum): + PASSED = enum.auto() + FAILED = enum.auto() + SKIPPED = enum.auto() + WARNED = enum.auto() + ERRORED = enum.auto() + + @classmethod + def from_string(cls, value: str): + value = value.lower() + if value.startswith(("failed", "fail")): + return cls.FAILED + if value.startswith(("passed", "pass", "success", "ok")): + return cls.PASSED + if value.startswith(("skipped", "skip", "ignore", "ignored")): + return cls.SKIPPED + if value.startswith("WARNING"): + return cls.WARNED + raise ValueError(f"Unknown test status `{value}`") + + def to_ansi_color(self): + if self == TestStatus.FAILED: + return "red" + if self == TestStatus.PASSED: + return "green" + return "yellow" + + +class TestCaseSource: + def __init__(self, filename, line=None): + self.filename = filename + self.line = line + + +class TestCase: + def __init__( # pylint: disable=too-many-arguments + self, + name, + status, + message=None, + stdout=None, + source=None, + duration=0, + exception=None, + ): + assert isinstance(status, TestStatus) + if status == TestStatus.ERRORED: + assert isinstance(exception, Exception) + self.name = name.strip() + self.status = status + self.message = message + self.stdout = stdout + self.source = source + self.duration = duration + self.exception = exception + + def humanize(self): + parts = [] + if self.source: + parts.append("%s:%d: " % (self.source.filename, self.source.line)) + parts.append(self.name) + if self.message: + parts.append(": " + self.message) + parts.extend( + [ + "\t", + "[%s]" % click.style(self.status.name, fg=self.status.to_ansi_color()), + ] + ) + return "".join(parts) + + +class TestSuite: + def __init__(self, env_name, test_name, finished=False, test_dir=None): + self.env_name = env_name + self.test_name = test_name + self.test_dir = test_dir + self.timestamp = 0 + self.duration = 0 + self._cases = [] + self._finished = finished + + @property + def cases(self): + return self._cases + + @property + def status(self): + for s in (TestStatus.ERRORED, TestStatus.FAILED): + if self.get_status_nums(s): + return s + if self._cases and any(c.status == TestStatus.PASSED for c in self._cases): + return TestStatus.PASSED + return TestStatus.SKIPPED + + def get_status_nums(self, status): + return len([True for c in self._cases if c.status == status]) + + def add_case(self, case: TestCase): + assert isinstance(case, TestCase) + self._cases.append(case) + + def is_finished(self): + return self._finished + + def on_start(self): + self.timestamp = time.time() + + def on_finish(self): + if self.is_finished(): + return + self._finished = True + self.duration = time.time() - self.timestamp + + +class TestResult: + def __init__(self, project_dir): + self.project_dir = project_dir + self._suites = [] + + @property + def suites(self): + return self._suites + + def add_suite(self, suite): + assert isinstance(suite, TestSuite) + self._suites.append(suite) + + @property + def duration(self): + return functools.reduce(operator.add, [s.duration for s in self._suites]) + + @property + def case_nums(self): + return functools.reduce(operator.add, [len(s.cases) for s in self._suites]) + + @property + def is_errored(self): + return any(s.status == TestStatus.ERRORED for s in self._suites) + + def get_status_nums(self, status): + return functools.reduce( + operator.add, [s.get_status_nums(status) for s in self._suites] + ) diff --git a/platformio/test/runners/__init__.py b/platformio/test/runners/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/test/runners/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py new file mode 100644 index 0000000..e8919f8 --- /dev/null +++ b/platformio/test/runners/base.py @@ -0,0 +1,215 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.exception import ReturnErrorCode +from platformio.platform.factory import PlatformFactory +from platformio.test.exception import UnitTestSuiteError +from platformio.test.result import TestCase, TestStatus +from platformio.test.runners.readers.native import NativeTestOutputReader +from platformio.test.runners.readers.serial import SerialTestOutputReader + +CTX_META_TEST_IS_RUNNING = __name__ + ".test_running" +CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name" + + +class TestRunnerOptions: # pylint: disable=too-many-instance-attributes + def __init__( # pylint: disable=too-many-arguments + self, + verbose=0, + without_building=False, + without_uploading=False, + without_testing=False, + without_debugging=True, + upload_port=None, + test_port=None, + no_reset=False, + monitor_rts=None, + monitor_dtr=None, + program_args=None, + ): + self.verbose = verbose + self.without_building = without_building + self.without_uploading = without_uploading + self.without_testing = without_testing + self.without_debugging = without_debugging + self.upload_port = upload_port + self.test_port = test_port + self.no_reset = no_reset + self.monitor_rts = monitor_rts + self.monitor_dtr = monitor_dtr + self.program_args = program_args + + +class TestRunnerBase: + NAME = None + EXTRA_LIB_DEPS = None + TESTCASE_PARSE_RE = None + + def __init__(self, test_suite, project_config, options=None): + self.test_suite = test_suite + self.options = options + self.project_config = project_config + self.platform = PlatformFactory.from_env( + self.test_suite.env_name, + autoinstall=True, + ) + self.cmd_ctx = None + self._testing_output_buffer = "" + + @property + def name(self): + return self.__class__.__name__.replace("TestRunner", "").lower() + + def get_test_speed(self): + return int( + self.project_config.get(f"env:{self.test_suite.env_name}", "test_speed") + ) + + def get_test_port(self): + return self.options.test_port or self.project_config.get( + f"env:{self.test_suite.env_name}", "test_port" + ) + + def start(self, cmd_ctx): + # setup command context + self.cmd_ctx = cmd_ctx + self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True + if self.test_suite.test_name != "*": + self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_suite.test_name + + self.test_suite.on_start() + try: + self.setup() + for stage in ("building", "uploading", "testing"): + getattr(self, f"stage_{stage}")() + if self.options.verbose: + click.echo() + except Exception as exc: # pylint: disable=broad-except + click.secho(str(exc), fg="red", err=True) + self.test_suite.add_case( + TestCase( + name=f"{self.test_suite.env_name}:{self.test_suite.test_name}", + status=TestStatus.ERRORED, + exception=exc, + ) + ) + finally: + self.test_suite.on_finish() + self.teardown() + + def setup(self): + pass + + def stage_building(self): + if self.options.without_building: + return None + # run "building" once at the "uploading" stage for the embedded target + if not self.options.without_uploading and self.platform.is_embedded(): + return None + click.secho("Building...", bold=True) + targets = ["__test"] + if not self.options.without_debugging: + targets.append("__debug") + if self.platform.is_embedded(): + targets.append("checkprogsize") + try: + return self.run_project_targets(targets) + except ReturnErrorCode as exc: + raise UnitTestSuiteError( + "Building stage has failed, see errors above. " + "Use `pio test -vvv` option to enable verbose output." + ) from exc + + def stage_uploading(self): + is_embedded = self.platform.is_embedded() + if self.options.without_uploading or not is_embedded: + return None + click.secho( + "Building & Uploading..." if is_embedded else "Uploading...", bold=True + ) + targets = ["upload"] + if self.options.without_building: + targets.append("nobuild") + else: + targets.append("__test") + if not self.options.without_debugging: + targets.append("__debug") + try: + return self.run_project_targets(targets) + except ReturnErrorCode as exc: + raise UnitTestSuiteError( + "Uploading stage has failed, see errors above. " + "Use `pio test -vvv` option to enable verbose output." + ) from exc + + def stage_testing(self): + if self.options.without_testing: + return None + click.secho("Testing...", bold=True) + test_port = self.get_test_port() + native_conds = [ + not self.platform.is_embedded() + and (not test_port or "://" not in test_port), + self.project_config.get( + f"env:{self.test_suite.env_name}", "test_testing_command" + ), + ] + reader = ( + NativeTestOutputReader(self) + if any(native_conds) + else SerialTestOutputReader(self) + ) + return reader.begin() + + def teardown(self): + pass + + def run_project_targets(self, targets): + # pylint: disable=import-outside-toplevel + from platformio.run.cli import cli as run_cmd + + assert self.cmd_ctx + return self.cmd_ctx.invoke( + run_cmd, + project_conf=self.project_config.path, + upload_port=self.options.upload_port, + verbose=self.options.verbose > 2, + silent=self.options.verbose < 2, + environment=[self.test_suite.env_name], + disable_auto_clean="nobuild" in targets, + target=targets, + ) + + def configure_build_env(self, env): + """ + Configure SCons build environment + Called in "builder/tools/piotest" tool + """ + return env + + def on_testing_data_output(self, data): + if isinstance(data, bytes): + data = data.decode("utf8", "ignore") + self._testing_output_buffer += data + self._testing_output_buffer = self._testing_output_buffer.replace("\r", "") + while "\n" in self._testing_output_buffer: + nl_pos = self._testing_output_buffer.index("\n") + line = self._testing_output_buffer[: nl_pos + 1] + self._testing_output_buffer = self._testing_output_buffer[nl_pos + 1 :] + self.on_testing_line_output(line) + + def on_testing_line_output(self, line): + click.echo(line, nl=False) diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py new file mode 100644 index 0000000..dd30bc9 --- /dev/null +++ b/platformio/test/runners/doctest.py @@ -0,0 +1,121 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from platformio.test.result import TestCase, TestCaseSource, TestStatus +from platformio.test.runners.base import TestRunnerBase + + +class DoctestTestCaseParser: + def __init__(self): + self._tmp_tc = None + self._name_tokens = [] + + def parse(self, line): + if self.is_divider(line): + return self._on_divider() + if not self._tmp_tc or line.strip().startswith("[doctest]"): + return None + + self._tmp_tc.stdout += line + line = line.strip() + + # source + if not self._tmp_tc.source and line: + self._tmp_tc.source = self.parse_source(line) + return None + + # name + if not self._tmp_tc.name: + if line: + self._name_tokens.append(line) + return None + self._tmp_tc.name = self.parse_name(self._name_tokens) + return None + + if self._tmp_tc.status != TestStatus.FAILED: + self._parse_assert(line) + + return None + + @staticmethod + def is_divider(line): + line = line.strip() + return line.startswith("===") and line.endswith("===") + + def _on_divider(self): + test_case = None + if self._tmp_tc: + test_case = TestCase( + name=self._tmp_tc.name.strip(), + status=self._tmp_tc.status, + message=(self._tmp_tc.message or "").strip() or None, + source=self._tmp_tc.source, + stdout=self._tmp_tc.stdout.strip(), + ) + + self._tmp_tc = TestCase("", TestStatus.PASSED, stdout="") + self._name_tokens = [] + return test_case + + @staticmethod + def parse_source(line): + if not line.endswith(":"): + return None + filename, line = line[:-1].rsplit(":", 1) + return TestCaseSource(filename, int(line)) + + @staticmethod + def parse_name(tokens): + cleaned_tokens = [] + for token in tokens: + if token.startswith("TEST ") and ":" in token: + token = token[token.index(":") + 1 :] + cleaned_tokens.append(token.strip()) + return "/".join(cleaned_tokens) + + def _parse_assert(self, line): + status_tokens = [ + (TestStatus.FAILED, "ERROR"), + (TestStatus.FAILED, "FATAL ERROR"), + (TestStatus.WARNED, "WARNING"), + ] + for status, token in status_tokens: + index = line.find(": %s:" % token) + if index == -1: + continue + self._tmp_tc.status = status + self._tmp_tc.message = line[index + len(token) + 3 :].strip() or None + + +class DoctestTestRunner(TestRunnerBase): + EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.9"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._tc_parser = DoctestTestCaseParser() + + def on_testing_line_output(self, line): + if self.options.verbose: + click.echo(line, nl=False) + + test_case = self._tc_parser.parse(line) + if test_case: + self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) + + if "[doctest] Status:" in line: + self.test_suite.on_finish() diff --git a/platformio/test/runners/factory.py b/platformio/test/runners/factory.py new file mode 100644 index 0000000..ecb6f88 --- /dev/null +++ b/platformio/test/runners/factory.py @@ -0,0 +1,67 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import os +import re + +from platformio.compat import load_python_module +from platformio.exception import UserSideException +from platformio.project.config import ProjectConfig +from platformio.test.result import TestSuite +from platformio.test.runners.base import TestRunnerBase, TestRunnerOptions + + +class TestRunnerFactory: + @staticmethod + def get_clsname(name): + name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I) + return "%sTestRunner" % name.lower().capitalize() + + @classmethod + def new(cls, test_suite, project_config, options=None) -> TestRunnerBase: + assert isinstance(test_suite, TestSuite) + assert isinstance(project_config, ProjectConfig) + if options: + assert isinstance(options, TestRunnerOptions) + test_framework = project_config.get( + f"env:{test_suite.env_name}", "test_framework" + ) + module_name = f"platformio.test.runners.{test_framework}" + runner_cls = None + if test_framework == "custom": + test_dir = project_config.get("platformio", "test_dir") + custom_runner_path = os.path.join(test_dir, "test_custom_runner.py") + test_name = test_suite.test_name if test_suite.test_name != "*" else None + while test_name: + if os.path.isfile( + os.path.join(test_dir, test_name, "test_custom_runner.py") + ): + custom_runner_path = os.path.join( + test_dir, test_name, "test_custom_runner.py" + ) + break + test_name = os.path.dirname(test_name) # parent dir + + try: + mod = load_python_module(module_name, custom_runner_path) + except (FileNotFoundError, ImportError) as exc: + raise UserSideException( + "Could not find custom test runner " + f"by this path -> {custom_runner_path}" + ) from exc + else: + mod = importlib.import_module(module_name) + runner_cls = getattr(mod, cls.get_clsname(test_framework)) + return runner_cls(test_suite, project_config, options) diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py new file mode 100644 index 0000000..8a4d1b1 --- /dev/null +++ b/platformio/test/runners/googletest.py @@ -0,0 +1,109 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import click + +from platformio.test.result import TestCase, TestCaseSource, TestStatus +from platformio.test.runners.base import TestRunnerBase + + +class GoogletestTestCaseParser: + # Examples: + # [ RUN ] FooTest.Bar + # ... + # [ FAILED ] FooTest.Bar (0 ms) + STATUS__NAME_RE = r"^\[\s+(?P[A-Z]+)\s+\]\s+(?P[^\(\s]+)" + + # Examples: + # [ RUN ] FooTest.Bar + # test/test_gtest/test_main.cpp:26: Failure + # Y:\core\examples\unit-testing\googletest\test\test_gtest\test_main.cpp:26: Failure + SOURCE_MESSAGE_RE = r"^(?P.+):(?P\d+):(?P.*)$" + + def __init__(self): + self._tmp_tc = None + + def parse(self, line): + if self._tmp_tc: + self._tmp_tc.stdout += line + return self._parse_test_case(line) + + def _parse_test_case(self, line): + status, name = self._parse_status_and_name(line) + if status == "RUN": + self._tmp_tc = TestCase(name, TestStatus.PASSED, stdout=line) + return None + if not status or not self._tmp_tc: + return None + source, message = self._parse_source_and_message(self._tmp_tc.stdout) + test_case = TestCase( + name=self._tmp_tc.name, + status=TestStatus.from_string(status), + message=message, + source=source, + stdout=self._tmp_tc.stdout.strip(), + ) + self._tmp_tc = None + return test_case + + def _parse_status_and_name(self, line): + result = (None, None) + line = line.strip() + if not line.startswith("["): + return result + match = re.search(self.STATUS__NAME_RE, line) + if not match: + return result + return match.group("status"), match.group("name") + + def _parse_source_and_message(self, stdout): + for line in stdout.split("\n"): + line = line.strip() + if not line: + continue + match = re.search(self.SOURCE_MESSAGE_RE, line) + if not match: + continue + return ( + TestCaseSource( + match.group("source_file"), int(match.group("source_line")) + ), + (match.group("message") or "").strip() or None, + ) + return (None, None) + + +class GoogletestTestRunner(TestRunnerBase): + EXTRA_LIB_DEPS = ["google/googletest@^1.12.1"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._tc_parser = GoogletestTestCaseParser() + os.environ["GTEST_COLOR"] = "no" # disable ANSI symbols + + def on_testing_line_output(self, line): + if self.options.verbose: + click.echo(line, nl=False) + + test_case = self._tc_parser.parse(line) + if test_case: + self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) + + if "Global test environment tear-down" in line: + self.test_suite.on_finish() diff --git a/platformio/test/runners/readers/__init__.py b/platformio/test/runners/readers/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/platformio/test/runners/readers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/test/runners/readers/native.py b/platformio/test/runners/readers/native.py new file mode 100644 index 0000000..981febc --- /dev/null +++ b/platformio/test/runners/readers/native.py @@ -0,0 +1,136 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import os +import signal +import subprocess +import time + +from platformio.compat import ( + IS_WINDOWS, + aio_get_running_loop, + get_filesystem_encoding, + get_locale_encoding, +) +from platformio.project.helpers import load_build_metadata +from platformio.test.exception import UnitTestError + +EXITING_TIMEOUT = 5 # seconds + + +class ProgramProcessProtocol(asyncio.SubprocessProtocol): + def __init__(self, test_runner, exit_future): + self.test_runner = test_runner + self.exit_future = exit_future + self._exit_timer = None + + def pipe_data_received(self, _, data): + try: + data = data.decode(get_locale_encoding() or get_filesystem_encoding()) + except UnicodeDecodeError: + data = data.decode("latin-1") + self.test_runner.on_testing_data_output(data) + if self.test_runner.test_suite.is_finished(): + self._exit_timer = aio_get_running_loop().call_later( + EXITING_TIMEOUT, self._stop_testing + ) + + def process_exited(self): + self._stop_testing() + + def _stop_testing(self): + if not self.exit_future.done(): + self.exit_future.set_result(True) + if self._exit_timer: + self._exit_timer.cancel() + + +class NativeTestOutputReader: + def __init__(self, test_runner): + self.test_runner = test_runner + self.aio_loop = ( + asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.new_event_loop() + ) + asyncio.set_event_loop(self.aio_loop) + + def get_testing_command(self): + custom_testing_command = self.test_runner.project_config.get( + f"env:{self.test_runner.test_suite.env_name}", "test_testing_command" + ) + if custom_testing_command: + return custom_testing_command + build_dir = self.test_runner.project_config.get("platformio", "build_dir") + cmd = [ + os.path.join( + build_dir, + self.test_runner.test_suite.env_name, + "program.exe" if IS_WINDOWS else "program", + ) + ] + # if user changed PROGNAME + if not os.path.exists(cmd[0]): + build_data = load_build_metadata( + os.getcwd(), self.test_runner.test_suite.env_name, cache=True + ) + if build_data: + cmd[0] = build_data["prog_path"] + if self.test_runner.options.program_args: + cmd.extend(self.test_runner.options.program_args) + return cmd + + async def gather_results(self): + exit_future = asyncio.Future(loop=self.aio_loop) + transport, _ = await self.aio_loop.subprocess_exec( + lambda: ProgramProcessProtocol(self.test_runner, exit_future), + *self.get_testing_command(), + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + await exit_future + last_return_code = transport.get_returncode() + transport.close() + + # wait until subprocess will be killed + start = time.time() + while ( + start > (time.time() - EXITING_TIMEOUT) + and transport.get_returncode() is None + ): + await asyncio.sleep(0.5) + + if last_return_code: + self.raise_for_status(last_return_code) + + @staticmethod + def raise_for_status(return_code): + try: + sig = signal.Signals(abs(return_code)) + try: + signal_description = signal.strsignal(sig) + except AttributeError: + signal_description = "" + raise UnitTestError( + f"Program received signal {sig.name} ({signal_description})" + ) + except ValueError as exc: + raise UnitTestError("Program errored with %d code" % return_code) from exc + + def begin(self): + try: + self.aio_loop.run_until_complete(self.gather_results()) + finally: + self.aio_loop.run_until_complete(self.aio_loop.shutdown_asyncgens()) + self.aio_loop.close() diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py new file mode 100644 index 0000000..a9732d7 --- /dev/null +++ b/platformio/test/runners/readers/serial.py @@ -0,0 +1,81 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + +import click +import serial + +from platformio.device.finder import SerialPortFinder +from platformio.exception import UserSideException + + +class SerialTestOutputReader: + SERIAL_TIMEOUT = 600 + + def __init__(self, test_runner): + self.test_runner = test_runner + + def begin(self): + click.echo( + "If you don't see any output for the first 10 secs, " + "please reset board (press reset button)" + ) + click.echo() + + try: + ser = serial.serial_for_url( + self.resolve_test_port(), + do_not_open=True, + baudrate=self.test_runner.get_test_speed(), + timeout=self.SERIAL_TIMEOUT, + ) + ser.rts = self.test_runner.options.monitor_rts + ser.dtr = self.test_runner.options.monitor_dtr + ser.open() + except serial.SerialException as exc: + click.secho(str(exc), fg="red", err=True) + return + + if not self.test_runner.options.no_reset: + ser.flushInput() + ser.setDTR(False) + ser.setRTS(False) + sleep(0.1) + ser.setDTR(True) + ser.setRTS(True) + sleep(0.1) + + while not self.test_runner.test_suite.is_finished(): + self.test_runner.on_testing_data_output(ser.read(ser.in_waiting or 1)) + ser.close() + + def resolve_test_port(self): + project_options = self.test_runner.project_config.items( + env=self.test_runner.test_suite.env_name, as_dict=True + ) + port = SerialPortFinder( + board_config=self.test_runner.platform.board_config( + project_options["board"] + ), + upload_protocol=project_options.get("upload_protocol"), + ensure_ready=True, + verbose=self.test_runner.options.verbose, + ).find(initial_port=self.test_runner.get_test_port()) + if port: + return port + raise UserSideException( + "Please specify `test_port` for environment or use " + "global `--test-port` option." + ) diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py new file mode 100644 index 0000000..935a132 --- /dev/null +++ b/platformio/test/runners/unity.py @@ -0,0 +1,303 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import string +from pathlib import Path + +import click + +from platformio.test.exception import UnitTestSuiteError +from platformio.test.result import TestCase, TestCaseSource, TestStatus +from platformio.test.runners.base import TestRunnerBase +from platformio.util import strip_ansi_codes + + +class UnityTestRunner(TestRunnerBase): + EXTRA_LIB_DEPS = ["throwtheswitch/Unity@^2.5.2"] + + # Examples: + # test/test_foo.cpp:44:test_function_foo:FAIL: Expected 32 Was 33 + # test/group/test_foo/test_main.cpp:5:test::dummy:FAIL: Expression Evaluated To FALSE + TESTCASE_PARSE_RE = re.compile( + r"(?P[^:]+):(?P\d+):(?P[^\s]+):" + r"(?PPASS|IGNORE|FAIL)(:\s*(?P.+)$)?" + ) + + UNITY_CONFIG_H = """ +#ifndef UNITY_CONFIG_H +#define UNITY_CONFIG_H + +#ifndef NULL +#ifndef __cplusplus +#define NULL (void*)0 +#else +#define NULL 0 +#endif +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + +void unityOutputStart(unsigned long); +void unityOutputChar(unsigned int); +void unityOutputFlush(void); +void unityOutputComplete(void); + +#define UNITY_OUTPUT_START() unityOutputStart((unsigned long) $baudrate) +#define UNITY_OUTPUT_CHAR(c) unityOutputChar(c) +#define UNITY_OUTPUT_FLUSH() unityOutputFlush() +#define UNITY_OUTPUT_COMPLETE() unityOutputComplete() + +#ifdef __cplusplus +} +#endif /* extern "C" */ + +#endif /* UNITY_CONFIG_H */ + +""" + + UNITY_CONFIG_C = """ +#include + +#if !defined(UNITY_WEAK_ATTRIBUTE) && !defined(UNITY_WEAK_PRAGMA) +# if defined(__GNUC__) || defined(__ghs__) /* __GNUC__ includes clang */ +# if !(defined(__WIN32__) && defined(__clang__)) && !defined(__TMS470__) +# define UNITY_WEAK_ATTRIBUTE __attribute__((weak)) +# endif +# endif +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + +#ifdef UNITY_WEAK_ATTRIBUTE + UNITY_WEAK_ATTRIBUTE void setUp(void) { } + UNITY_WEAK_ATTRIBUTE void tearDown(void) { } + UNITY_WEAK_ATTRIBUTE void suiteSetUp(void) { } + UNITY_WEAK_ATTRIBUTE int suiteTearDown(int num_failures) { return num_failures; } +#elif defined(UNITY_WEAK_PRAGMA) + #pragma weak setUp + void setUp(void) { } + #pragma weak tearDown + void tearDown(void) { } + #pragma weak suiteSetUp + void suiteSetUp(void) { } + #pragma weak suiteTearDown + int suiteTearDown(int num_failures) { return num_failures; } +#endif + +#ifdef __cplusplus +} +#endif /* extern "C" */ + +$framework_config_code + """ + + UNITY_FRAMEWORK_CONFIG = dict( + native=dict( + code=""" +#include +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } +void unityOutputChar(unsigned int c) { putchar(c); } +void unityOutputFlush(void) { fflush(stdout); } +void unityOutputComplete(void) { } + """, + language="c", + ), + arduino=dict( + code=""" +#include +void unityOutputStart(unsigned long baudrate) { Serial.begin(baudrate); } +void unityOutputChar(unsigned int c) { Serial.write(c); } +void unityOutputFlush(void) { Serial.flush(); } +void unityOutputComplete(void) { Serial.end(); } + """, + language="cpp", + ), + mbed=dict( + code=""" +#include +#if MBED_MAJOR_VERSION == 6 +UnbufferedSerial pc(USBTX, USBRX); +#else +RawSerial pc(USBTX, USBRX); +#endif +void unityOutputStart(unsigned long baudrate) { pc.baud(baudrate); } +void unityOutputChar(unsigned int c) { +#if MBED_MAJOR_VERSION == 6 + pc.write(&c, 1); +#else + pc.putc(c); +#endif +} +void unityOutputFlush(void) { } +void unityOutputComplete(void) { } + """, + language="cpp", + ), + espidf=dict( + code=""" +#include +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } +void unityOutputChar(unsigned int c) { putchar(c); } +void unityOutputFlush(void) { fflush(stdout); } +void unityOutputComplete(void) { } + """, + language="c", + ), + zephyr=dict( + code=""" +#include +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } +void unityOutputChar(unsigned int c) { printk("%c", c); } +void unityOutputFlush(void) { } +void unityOutputComplete(void) { } + """, + language="c", + ), + legacy_custom_transport=dict( + code=""" +#include +void unityOutputStart(unsigned long baudrate) { unittest_uart_begin(); } +void unityOutputChar(unsigned int c) { unittest_uart_putchar(c); } +void unityOutputFlush(void) { unittest_uart_flush(); } +void unityOutputComplete(void) { unittest_uart_end(); } + """, + language="cpp", + ), + ) + + def __init__(self, *args, **kwargs): + """Delete when Unity > 2.5.2 is released""" + super().__init__(*args, **kwargs) + + def get_unity_framework_config(self): + if not self.platform.is_embedded(): + return self.UNITY_FRAMEWORK_CONFIG["native"] + if ( + self.project_config.get( + f"env:{self.test_suite.env_name}", "test_transport", None + ) + == "custom" + ): + framework = "legacy_custom_transport" + else: + framework = ( + self.project_config.get(f"env:{self.test_suite.env_name}", "framework") + or [None] + )[0] + if framework and framework in self.UNITY_FRAMEWORK_CONFIG: + return self.UNITY_FRAMEWORK_CONFIG[framework] + raise UnitTestSuiteError( + f"Could not find Unity configuration for the `{framework}` framework.\n" + "Learn how to create a custom Unity configuration at" + "https://docs.platformio.org/en/latest/advanced/" + "unit-testing/frameworks/unity.html" + ) + + def configure_build_env(self, env): + env.Append(CPPDEFINES=["UNITY_INCLUDE_CONFIG_H"]) + if self.custom_unity_config_exists(): + return env + env.Replace( + UNITY_CONFIG_DIR=os.path.join("$BUILD_DIR", "unity_config"), + BUILD_UNITY_CONFIG_DIR=os.path.join("$BUILD_DIR", "unity_config_build"), + ) + env.Prepend(CPPPATH=["$UNITY_CONFIG_DIR"]) + self.generate_unity_extras(env.subst("$UNITY_CONFIG_DIR")) + env.BuildSources("$BUILD_UNITY_CONFIG_DIR", "$UNITY_CONFIG_DIR") + return env + + def custom_unity_config_exists(self): + test_dir = self.project_config.get("platformio", "test_dir") + config_fname = "unity_config.h" + if os.path.isfile(os.path.join(test_dir, config_fname)): + return True + test_name = ( + self.test_suite.test_name if self.test_suite.test_name != "*" else None + ) + while test_name: + if os.path.isfile(os.path.join(test_dir, test_name, config_fname)): + return True + test_name = os.path.dirname(test_name) # parent dir + return False + + def generate_unity_extras(self, dst_dir): + dst_dir = Path(dst_dir) + if not dst_dir.is_dir(): + dst_dir.mkdir(parents=True) + unity_h = dst_dir / "unity_config.h" + if not unity_h.is_file(): + unity_h.write_text( + string.Template(self.UNITY_CONFIG_H) + .substitute(baudrate=self.get_test_speed()) + .strip() + + "\n", + encoding="utf8", + ) + framework_config = self.get_unity_framework_config() + unity_c = dst_dir / ("unity_config.%s" % framework_config.get("language", "c")) + if not unity_c.is_file(): + unity_c.write_text( + string.Template(self.UNITY_CONFIG_C) + .substitute(framework_config_code=framework_config["code"]) + .strip() + + "\n", + encoding="utf8", + ) + + def on_testing_line_output(self, line): + if self.options.verbose: + click.echo(line, nl=False) + line = strip_ansi_codes(line or "").strip() + if not line: + return + + test_case = self.parse_test_case(line) + if test_case: + self.test_suite.add_case(test_case) + if not self.options.verbose: + click.echo(test_case.humanize()) + + if all(s in line for s in ("Tests", "Failures", "Ignored")): + self.test_suite.on_finish() + + def parse_test_case(self, line): + if not self.TESTCASE_PARSE_RE: + raise NotImplementedError() + line = line.strip() + if not line: + return None + match = self.TESTCASE_PARSE_RE.search(line) + if not match: + return None + data = match.groupdict() + source = None + if "source_file" in data: + source = TestCaseSource( + filename=data["source_file"], line=int(data.get("source_line")) + ) + return TestCase( + name=data.get("name").strip(), + status=TestStatus.from_string(data.get("status")), + message=(data.get("message") or "").strip() or None, + stdout=line, + source=source, + ) diff --git a/platformio/util.py b/platformio/util.py new file mode 100644 index 0000000..004ac5c --- /dev/null +++ b/platformio/util.py @@ -0,0 +1,208 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import functools +import math +import platform +import re +import shutil +import time + +import click + +from platformio import __version__ + +# pylint: disable=unused-import +from platformio.device.list.util import list_serial_ports as get_serial_ports +from platformio.fs import cd, load_json +from platformio.proc import exec_command + +# pylint: enable=unused-import + +# also export list_serial_ports as get_serialports to be +# backward compatibility with arduinosam versions 3.9.0 to 3.5.0 (and possibly others) +get_serialports = get_serial_ports + + +class memoized: + def __init__(self, expire=0): + expire = str(expire) + if expire.isdigit(): + expire = "%ss" % int((int(expire) / 1000)) + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert expire.endswith(tuple(tdmap)) + self.expire = int(tdmap[expire[-1]] * int(expire[:-1])) + self.cache = {} + + def __call__(self, func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + key = str(args) + str(kwargs) + if key not in self.cache or ( + self.expire > 0 and self.cache[key][0] < time.time() - self.expire + ): + self.cache[key] = (time.time(), func(*args, **kwargs)) + return self.cache[key][1] + + wrapper.reset = self._reset + return wrapper + + def _reset(self): + self.cache.clear() + + +class throttle: + def __init__(self, threshhold): + self.threshhold = threshhold # milliseconds + self.last = 0 + + def __call__(self, func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + diff = int(round((time.time() - self.last) * 1000)) + if diff < self.threshhold: + time.sleep((self.threshhold - diff) * 0.001) + self.last = time.time() + return func(*args, **kwargs) + + return wrapper + + +# Retry: Begin + + +class RetryException(Exception): + pass + + +class RetryNextException(RetryException): + pass + + +class RetryStopException(RetryException): + pass + + +class retry: + RetryNextException = RetryNextException + RetryStopException = RetryStopException + + def __init__(self, timeout=0, step=0.25): + self.timeout = timeout + self.step = step + + def __call__(self, func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + elapsed = 0 + while True: + try: + return func(*args, **kwargs) + except self.RetryNextException: + pass + if elapsed >= self.timeout: + raise self.RetryStopException() + elapsed += self.step + time.sleep(self.step) + + return wrapper + + +# Retry: End + + +def singleton(cls): + """From PEP-318 http://www.python.org/dev/peps/pep-0318/#examples""" + _instances = {} + + def get_instance(*args, **kwargs): + if cls not in _instances: + _instances[cls] = cls(*args, **kwargs) + return _instances[cls] + + return get_instance + + +def get_systype(): + system = platform.system().lower() + arch = platform.machine().lower() + if system == "windows": + if not arch: # issue #4353 + arch = "x86_" + platform.architecture()[0] + if "x86" in arch: + arch = "amd64" if "64" in arch else "x86" + return "%s_%s" % (system, arch) if arch else system + + +def pioversion_to_intstr(): + """Legacy for framework-zephyr/scripts/platformio/platformio-build-pre.py""" + vermatch = re.match(r"^([\d\.]+)", __version__) + assert vermatch + return [int(i) for i in vermatch.group(1).split(".")[:3]] + + +def items_to_list(items): + if isinstance(items, list): + return items + return [i.strip() for i in items.split(",") if i.strip()] + + +def items_in_list(needle, haystack): + needle = items_to_list(needle) + haystack = items_to_list(haystack) + if "*" in needle or "*" in haystack: + return True + return set(needle) & set(haystack) + + +def parse_datetime(datestr): + if "T" in datestr and "Z" in datestr: + return datetime.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") + return datetime.datetime.strptime(datestr) + + +def merge_dicts(d1, d2, path=None): + if path is None: + path = [] + for key in d2: + if key in d1 and isinstance(d1[key], dict) and isinstance(d2[key], dict): + merge_dicts(d1[key], d2[key], path + [str(key)]) + else: + d1[key] = d2[key] + return d1 + + +def print_labeled_bar(label, is_error=False, fg=None, sep="="): + terminal_width = shutil.get_terminal_size().columns + width = len(click.unstyle(label)) + half_line = sep * int((terminal_width - width - 2) / 2) + click.secho("%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) + + +def humanize_duration_time(duration): + if duration is None: + return duration + duration = duration * 1000 + tokens = [] + for multiplier in (3600000, 60000, 1000, 1): + fraction = math.floor(duration / multiplier) + tokens.append(int(round(duration) if multiplier == 1 else fraction)) + duration -= fraction * multiplier + return "{:02d}:{:02d}:{:02d}.{:03d}".format(*tokens) + + +def strip_ansi_codes(text): + # pylint: disable=protected-access + return click._compat.strip_ansi(text) diff --git a/scripts/docspregen.py b/scripts/docspregen.py new file mode 100644 index 0000000..a4d56c0 --- /dev/null +++ b/scripts/docspregen.py @@ -0,0 +1,1216 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import os +import sys +import tempfile +from urllib.parse import ParseResult, urlparse, urlunparse + +sys.path.append("..") + +import click # noqa: E402 + +from platformio import fs # noqa: E402 +from platformio.package.manager.platform import PlatformPackageManager # noqa: E402 +from platformio.platform.factory import PlatformFactory # noqa: E402 + + +RST_COPYRIGHT = """.. Copyright (c) 2014-present PlatformIO + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +SKIP_DEBUG_TOOLS = ["esp-bridge", "esp-builtin"] + +STATIC_FRAMEWORK_DATA = { + "arduino": { + "title": "Arduino", + "description": ( + "Arduino Wiring-based Framework allows writing cross-platform software " + "to control devices attached to a wide range of Arduino boards to " + "create all kinds of creative coding, interactive objects, spaces " + "or physical experiences." + ), + }, + "cmsis": { + "title": "CMSIS", + "description": ( + "Vendor-independent hardware abstraction layer for the Cortex-M processor series" + ), + }, + "freertos": { + "title": "FreeRTOS", + "description": ( + "FreeRTOS is a real-time operating system kernel for embedded devices " + "that has been ported to 40 microcontroller platforms." + ), + }, +} + +DOCS_ROOT_DIR = os.path.realpath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "docs") +) +REGCLIENT = PlatformPackageManager().get_registry_client_instance() + + +def reg_package_url(type_, owner, name): + if type_ == "library": + type_ = "libraries" + else: + type_ += "s" + return f"https://registry.platformio.org/{type_}/{owner}/{name}" + + +def campaign_url(url, source="platformio.org", medium="docs"): + data = urlparse(url) + query = data.query + if query: + query += "&" + query += "utm_source=%s&utm_medium=%s" % (source, medium) + return urlunparse( + ParseResult( + data.scheme, data.netloc, data.path, data.params, query, data.fragment + ) + ) + + +def install_platforms(): + print("Installing platforms...") + page = 1 + pm = PlatformPackageManager() + while True: + result = REGCLIENT.list_packages(qualifiers=dict(types=["platform"]), page=page) + for item in result["items"]: + spec = "%s/%s" % (item["owner"]["username"], item["name"]) + skip_conds = [ + item["owner"]["username"] != "platformio", + item["tier"] == "community", + ] + if all(skip_conds): + click.secho("Skip community platform: %s" % spec, fg="yellow") + continue + pm.install(spec, skip_dependencies=True) + page += 1 + if not result["items"] or result["page"] * result["limit"] >= result["total"]: + break + + +@functools.cache +def get_frameworks(): + items = {} + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for name, options in (p.frameworks or {}).items(): + if name in items: + continue + if name in STATIC_FRAMEWORK_DATA: + items[name] = dict( + name=name, + title=STATIC_FRAMEWORK_DATA[name]["title"], + description=STATIC_FRAMEWORK_DATA[name]["description"], + ) + continue + title = options.get("title") or name.title() + description = options.get("description") + if "package" in options: + regdata = REGCLIENT.get_package( + "tool", + p.packages[options["package"]].get("owner", "platformio"), + options["package"], + ) + title = regdata["title"] or title + description = regdata["description"] + items[name] = dict(name=name, title=title, description=description) + return sorted(items.values(), key=lambda item: item["name"]) + + +def is_compat_platform_and_framework(platform, framework): + p = PlatformFactory.new(platform) + return framework in (p.frameworks or {}).keys() + + +def generate_boards_table(boards, skip_columns=None): + columns = [ + ("Name", ":ref:`board_{platform}_{id}`"), + ("Platform", ":ref:`platform_{platform}`"), + ("Debug", "{debug}"), + ("MCU", "{mcu}"), + ("Frequency", "{f_cpu}MHz"), + ("Flash", "{rom}"), + ("RAM", "{ram}"), + ] + lines = [] + lines.append( + """ +.. list-table:: + :header-rows: 1 +""" + ) + + # add header + for name, template in columns: + if skip_columns and name in skip_columns: + continue + prefix = " * - " if name == "Name" else " - " + lines.append(prefix + name) + + for data in sorted(boards, key=lambda item: item["name"]): + has_onboard_debug = data.get("debug") and any( + t.get("onboard") for (_, t) in data["debug"]["tools"].items() + ) + debug = "No" + if has_onboard_debug: + debug = "On-board" + elif data.get("debug"): + debug = "External" + + variables = dict( + id=data["id"], + name=data["name"], + platform=data["platform"], + debug=debug, + mcu=data["mcu"].upper(), + f_cpu=int(data["fcpu"] / 1000000.0), + ram=fs.humanize_file_size(data["ram"]), + rom=fs.humanize_file_size(data["rom"]), + ) + + for name, template in columns: + if skip_columns and name in skip_columns: + continue + prefix = " * - " if name == "Name" else " - " + lines.append(prefix + template.format(**variables)) + + if lines: + lines.append("") + + return lines + + +def generate_frameworks_contents(frameworks): + if not frameworks: + return [] + lines = [] + lines.append( + """ +Frameworks +---------- +.. list-table:: + :header-rows: 1 + + * - Name + - Description""" + ) + known = set() + for framework in get_frameworks(): + known.add(framework["name"]) + if framework["name"] not in frameworks: + continue + lines.append( + """ + * - :ref:`framework_{name}` + - {description}""".format( + **framework + ) + ) + if set(frameworks) - known: + click.secho("Unknown frameworks %s " % (set(frameworks) - known), fg="red") + return lines + + +def generate_platforms_contents(platforms): + if not platforms: + return [] + lines = [] + lines.append( + """ +Platforms +--------- +.. list-table:: + :header-rows: 1 + + * - Name + - Description""" + ) + + for name in sorted(platforms): + p = PlatformFactory.new(name) + lines.append( + """ + * - :ref:`platform_{name}` + - {description}""".format( + name=p.name, description=p.description + ) + ) + return lines + + +def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): + if not skip_board_columns: + skip_board_columns = [] + skip_board_columns.append("Debug") + lines = [] + onboard_debug = [ + b + for b in boards + if b.get("debug") + and any(t.get("onboard") for (_, t) in b["debug"]["tools"].items()) + ] + external_debug = [b for b in boards if b.get("debug") and b not in onboard_debug] + if not onboard_debug and not external_debug: + return lines + + lines.append( + """ +Debugging +--------- + +:ref:`piodebug` - "1-click" solution for debugging with a zero configuration. + +.. contents:: + :local: +""" + ) + if extra_rst: + lines.append(".. include:: %s" % extra_rst) + + lines.append( + """ +Tools & Debug Probes +~~~~~~~~~~~~~~~~~~~~ + +Supported debugging tools are listed in "Debug" column. For more detailed +information, please scroll table by horizontal. +You can switch between debugging :ref:`debugging_tools` using +:ref:`projectconf_debug_tool` option in :ref:`projectconf`. + +.. warning:: + You will need to install debug tool drivers depending on your system. + Please click on compatible debug tool below for the further instructions. +""" + ) + + if onboard_debug: + lines.append( + """ +On-Board Debug Tools +^^^^^^^^^^^^^^^^^^^^ + +Boards listed below have on-board debug probe and **ARE READY** for debugging! +You do not need to use/buy external debug probe. +""" + ) + lines.extend( + generate_boards_table(onboard_debug, skip_columns=skip_board_columns) + ) + if external_debug: + lines.append( + """ +External Debug Tools +^^^^^^^^^^^^^^^^^^^^ + +Boards listed below are compatible with :ref:`piodebug` but **DEPEND ON** +external debug probe. They **ARE NOT READY** for debugging. +Please click on board name for the further details. +""" + ) + lines.extend( + generate_boards_table(external_debug, skip_columns=skip_board_columns) + ) + return lines + + +def generate_packages(platform, packages, is_embedded): + if not packages: + return + lines = [] + lines.append( + """ +Packages +-------- +""" + ) + lines.append( + """.. list-table:: + :header-rows: 1 + + * - Name + - Description""" + ) + for name, options in dict(sorted(packages.items())).items(): + package = REGCLIENT.get_package( + "tool", options.get("owner", "platformio"), name + ) + lines.append( + """ + * - `{name} <{url}>`__ + - {description}""".format( + name=package["name"], + url=reg_package_url( + "tool", package["owner"]["username"], package["name"] + ), + description=package["description"], + ) + ) + + if is_embedded: + lines.append( + """ +.. warning:: + **Linux Users**: + + * Install "udev" rules :ref:`platformio_udev_rules` + * Raspberry Pi users, please read this article + `Enable serial port on Raspberry Pi `__. +""" + ) + + if platform == "teensy": + lines.append( + """ + **Windows Users:** + + Teensy programming uses only Windows built-in HID + drivers. When Teensy is programmed to act as a USB Serial device, + Windows XP, Vista, 7 and 8 require `this serial driver + `_ + is needed to access the COM port your program uses. No special driver + installation is necessary on Windows 10. +""" + ) + else: + lines.append( + """ + **Windows Users:** + + Please check that you have a correctly installed USB driver from board + manufacturer +""" + ) + + return "\n".join(lines) + + +def generate_platform(pkg, rst_dir): + name = pkg.metadata.name + print("Processing platform: %s" % name) + + compatible_boards = [ + board + for board in PlatformPackageManager().get_installed_boards() + if name == board["platform"] + ] + + lines = [] + lines.append(RST_COPYRIGHT) + + p = PlatformFactory.new(name) + assert p.repository_url.endswith(".git") + github_url = p.repository_url[:-4] + registry_url = reg_package_url("platform", pkg.metadata.spec.owner, name) + + lines.append(".. _platform_%s:" % p.name) + lines.append("") + + lines.append(p.title) + lines.append("=" * len(p.title)) + lines.append("") + lines.append(":Registry:") + lines.append(" `%s <%s>`__" % (registry_url, registry_url)) + lines.append(":Configuration:") + lines.append(" :ref:`projectconf_env_platform` = ``%s``" % p.name) + lines.append("") + lines.append(p.description) + lines.append( + """ +For more detailed information please visit `vendor site <%s>`_.""" + % campaign_url(p.homepage) + ) + lines.append( + """ +.. contents:: Contents + :local: + :depth: 1 +""" + ) + + # + # Extra + # + if os.path.isfile(os.path.join(rst_dir, "%s_extra.rst" % name)): + lines.append(".. include:: %s_extra.rst" % p.name) + + # + # Examples + # + lines.append( + """ +Examples +-------- + +Examples are listed from `%s development platform repository <%s>`_: +""" + % (p.title, campaign_url("%s/tree/master/examples" % github_url)) + ) + examples_dir = os.path.join(p.get_dir(), "examples") + if os.path.isdir(examples_dir): + for eitem in os.listdir(examples_dir): + example_dir = os.path.join(examples_dir, eitem) + if not os.path.isdir(example_dir) or not os.listdir(example_dir): + continue + url = "%s/tree/master/examples/%s" % (github_url, eitem) + lines.append("* `%s <%s>`_" % (eitem, campaign_url(url))) + + # + # Debugging + # + if compatible_boards: + lines.extend( + generate_debug_contents( + compatible_boards, + skip_board_columns=["Platform"], + extra_rst="%s_debug.rst" % name + if os.path.isfile(os.path.join(rst_dir, "%s_debug.rst" % name)) + else None, + ) + ) + + # + # Development version of dev/platform + # + lines.append( + """ +Stable and upstream versions +---------------------------- + +You can switch between `stable releases <{github_url}/releases>`__ +of {title} development platform and the latest upstream version using +:ref:`projectconf_env_platform` option in :ref:`projectconf` as described below. + +Stable +~~~~~~ + +.. code-block:: ini + + ; Latest stable version, NOT recommended + ; Pin the version as shown below + [env:latest_stable] + platform = {name} + {board} + ; Specific version + [env:custom_stable] + platform = {name}@x.y.z + {board} +Upstream +~~~~~~~~ + +.. code-block:: ini + + [env:upstream_develop] + platform = {github_url}.git + {board}""".format( + name=p.name, + title=p.title, + github_url=github_url, + board="board = ...\n" if p.is_embedded() else "", + ) + ) + + # + # Packages + # + _packages_content = generate_packages(name, p.packages, p.is_embedded()) + if _packages_content: + lines.append(_packages_content) + + # + # Frameworks + # + compatible_frameworks = [] + for framework in get_frameworks(): + if is_compat_platform_and_framework(name, framework["name"]): + compatible_frameworks.append(framework["name"]) + lines.extend(generate_frameworks_contents(compatible_frameworks)) + + # + # Boards + # + if compatible_boards: + vendors = {} + for board in compatible_boards: + if board["vendor"] not in vendors: + vendors[board["vendor"]] = [] + vendors[board["vendor"]].append(board) + + lines.append( + """ +Boards +------ + +.. note:: + * You can list pre-configured boards by :ref:`cmd_boards` command + * For more detailed ``board`` information please scroll the tables below by + horizontally. +""" + ) + + for vendor, boards in sorted(vendors.items()): + lines.append(str(vendor)) + lines.append("~" * len(vendor)) + lines.extend(generate_boards_table(boards, skip_columns=["Platform"])) + + return "\n".join(lines) + + +def update_platform_docs(): + platforms_dir = os.path.join(DOCS_ROOT_DIR, "platforms") + for pkg in PlatformPackageManager().get_installed(): + rst_path = os.path.join(platforms_dir, "%s.rst" % pkg.metadata.name) + with open(rst_path, "w") as f: + f.write(generate_platform(pkg, platforms_dir)) + + +def generate_framework(type_, framework, rst_dir=None): + print("Processing framework: %s" % type_) + + compatible_platforms = [ + pkg + for pkg in PlatformPackageManager().get_installed() + if is_compat_platform_and_framework(pkg.metadata.name, type_) + ] + compatible_boards = [ + board + for board in PlatformPackageManager().get_installed_boards() + if type_ in board["frameworks"] + ] + + lines = [] + + lines.append(RST_COPYRIGHT) + lines.append(".. _framework_%s:" % type_) + lines.append("") + + lines.append(framework["title"]) + lines.append("=" * len(framework["title"])) + lines.append("") + lines.append(":Configuration:") + lines.append(" :ref:`projectconf_env_framework` = ``%s``" % type_) + lines.append("") + lines.append(framework["description"]) + lines.append( + """ +.. contents:: Contents + :local: + :depth: 1""" + ) + + # Extra + if os.path.isfile(os.path.join(rst_dir, "%s_extra.rst" % type_)): + lines.append(".. include:: %s_extra.rst" % type_) + + if compatible_platforms: + # Platforms + lines.extend( + generate_platforms_contents( + [pkg.metadata.name for pkg in compatible_platforms] + ) + ) + + # examples + lines.append( + """ +Examples +-------- +""" + ) + for pkg in compatible_platforms: + p = PlatformFactory.new(pkg) + lines.append( + "* `%s for %s <%s>`_" + % ( + framework["title"], + p.title, + campaign_url("%s/tree/master/examples" % p.repository_url[:-4]), + ) + ) + + # + # Debugging + # + if compatible_boards: + lines.extend( + generate_debug_contents( + compatible_boards, + extra_rst="%s_debug.rst" % type_ + if os.path.isfile(os.path.join(rst_dir, "%s_debug.rst" % type_)) + else None, + ) + ) + + # + # Boards + # + if compatible_boards: + vendors = {} + for board in compatible_boards: + if board["vendor"] not in vendors: + vendors[board["vendor"]] = [] + vendors[board["vendor"]].append(board) + lines.append( + """ +Boards +------ + +.. note:: + * You can list pre-configured boards by :ref:`cmd_boards` command + * For more detailed ``board`` information please scroll the tables below by horizontally. +""" + ) + for vendor, boards in sorted(vendors.items()): + lines.append(str(vendor)) + lines.append("~" * len(vendor)) + lines.extend(generate_boards_table(boards)) + return "\n".join(lines) + + +def update_framework_docs(): + frameworks_dir = os.path.join(DOCS_ROOT_DIR, "frameworks") + for framework in get_frameworks(): + name = framework["name"] + rst_path = os.path.join(frameworks_dir, "%s.rst" % name) + with open(rst_path, "w") as f: + f.write(generate_framework(name, framework, frameworks_dir)) + + +def update_boards(): + print("Updating boards...") + lines = [] + + lines.append(RST_COPYRIGHT) + lines.append(".. _boards:") + lines.append("") + + lines.append("Boards") + lines.append("======") + + lines.append( + """ +Rapid Embedded Development, Continuous and IDE integration in a few +steps with PlatformIO thanks to built-in project generator for the most +popular embedded boards and IDEs. + +.. note:: + * You can list pre-configured boards by :ref:`cmd_boards` command + * For more detailed ``board`` information please scroll tables below by horizontal. +""" + ) + + platforms = {} + installed_boards = PlatformPackageManager().get_installed_boards() + for data in installed_boards: + platform = data["platform"] + if platform in platforms: + platforms[platform].append(data) + else: + platforms[platform] = [data] + + for platform, boards in sorted(platforms.items()): + p = PlatformFactory.new(platform) + lines.append(p.title) + lines.append("-" * len(p.title)) + lines.append( + """ +.. toctree:: + :maxdepth: 1 + """ + ) + for board in sorted(boards, key=lambda item: item["name"]): + lines.append(" %s/%s" % (platform, board["id"])) + lines.append("") + + emboards_rst = os.path.join(DOCS_ROOT_DIR, "boards", "index.rst") + with open(emboards_rst, "w") as f: + f.write("\n".join(lines)) + + # individual board page + for data in installed_boards: + rst_path = os.path.join( + DOCS_ROOT_DIR, "boards", data["platform"], "%s.rst" % data["id"] + ) + if not os.path.isdir(os.path.dirname(rst_path)): + os.makedirs(os.path.dirname(rst_path)) + update_embedded_board(rst_path, data) + + +def update_embedded_board(rst_path, board): + platform = PlatformFactory.new(board["platform"]) + board_config = platform.board_config(board["id"]) + + board_manifest_url = platform.repository_url + assert board_manifest_url + if board_manifest_url.endswith(".git"): + board_manifest_url = board_manifest_url[:-4] + board_manifest_url += "/blob/master/boards/%s.json" % board["id"] + + variables = dict( + id=board["id"], + name=board["name"], + platform=board["platform"], + platform_description=platform.description, + url=campaign_url(board["url"]), + mcu=board_config.get("build", {}).get("mcu", ""), + mcu_upper=board["mcu"].upper(), + f_cpu=board["fcpu"], + f_cpu_mhz=int(int(board["fcpu"]) / 1000000), + ram=fs.humanize_file_size(board["ram"]), + rom=fs.humanize_file_size(board["rom"]), + vendor=board["vendor"], + board_manifest_url=board_manifest_url, + upload_protocol=board_config.get("upload.protocol", ""), + ) + + lines = [RST_COPYRIGHT] + lines.append(".. _board_{platform}_{id}:".format(**variables)) + lines.append("") + lines.append(board["name"]) + lines.append("=" * len(board["name"])) + lines.append( + """ +.. contents:: + +Hardware +-------- + +Platform :ref:`platform_{platform}`: {platform_description} + +.. list-table:: + + * - **Microcontroller** + - {mcu_upper} + * - **Frequency** + - {f_cpu_mhz:d}MHz + * - **Flash** + - {rom} + * - **RAM** + - {ram} + * - **Vendor** + - `{vendor} <{url}>`__ +""".format( + **variables + ) + ) + + # + # Configuration + # + lines.append( + """ +Configuration +------------- + +Please use ``{id}`` ID for :ref:`projectconf_env_board` option in :ref:`projectconf`: + +.. code-block:: ini + + [env:{id}] + platform = {platform} + board = {id} + +You can override default {name} settings per build environment using +``board_***`` option, where ``***`` is a JSON object path from +board manifest `{id}.json <{board_manifest_url}>`_. For example, +``board_build.mcu``, ``board_build.f_cpu``, etc. + +.. code-block:: ini + + [env:{id}] + platform = {platform} + board = {id} + + ; change microcontroller + board_build.mcu = {mcu} + + ; change MCU frequency + board_build.f_cpu = {f_cpu}L +""".format( + **variables + ) + ) + + # + # Uploading + # + upload_protocols = board_config.get("upload.protocols", []) + if len(upload_protocols) > 1: + lines.append( + """ +Uploading +--------- +%s supports the following uploading protocols: +""" + % board["name"] + ) + for protocol in sorted(upload_protocols): + lines.append("* ``%s``" % protocol) + lines.append( + """ +Default protocol is ``%s``""" + % variables["upload_protocol"] + ) + lines.append( + """ +You can change upload protocol using :ref:`projectconf_upload_protocol` option: + +.. code-block:: ini + + [env:{id}] + platform = {platform} + board = {id} + + upload_protocol = {upload_protocol} +""".format( + **variables + ) + ) + + # + # Debugging + # + lines.append("Debugging") + lines.append("---------") + if not board.get("debug"): + lines.append( + ":ref:`piodebug` currently does not support {name} board.".format( + **variables + ) + ) + else: + default_debug_tool = board_config.get_debug_tool_name() + has_onboard_debug = any( + t.get("onboard") for (_, t) in board["debug"]["tools"].items() + ) + lines.append( + """ +:ref:`piodebug` - "1-click" solution for debugging with a zero configuration. + +.. warning:: + You will need to install debug tool drivers depending on your system. + Please click on compatible debug tool below for the further + instructions and configuration information. + +You can switch between debugging :ref:`debugging_tools` using +:ref:`projectconf_debug_tool` option in :ref:`projectconf`. +""" + ) + if has_onboard_debug: + lines.append( + "{name} has on-board debug probe and **IS READY** for " + "debugging. You don't need to use/buy external debug probe.".format( + **variables + ) + ) + else: + lines.append( + "{name} does not have on-board debug probe and **IS NOT " + "READY** for debugging. You will need to use/buy one of " + "external probe listed below.".format(**variables) + ) + lines.append( + """ +.. list-table:: + :header-rows: 1 + + * - Compatible Tools + - On-board + - Default""" + ) + for tool_name, tool_data in sorted(board["debug"]["tools"].items()): + lines.append( + """ * - {tool} + - {onboard} + - {default}""".format( + tool=f"``{tool_name}``" + if tool_name in SKIP_DEBUG_TOOLS + else f":ref:`debugging_tool_{tool_name}`", + onboard="Yes" if tool_data.get("onboard") else "", + default="Yes" if tool_name == default_debug_tool else "", + ) + ) + + if board["frameworks"]: + lines.extend(generate_frameworks_contents(board["frameworks"])) + + with open(rst_path, "w") as f: + f.write("\n".join(lines)) + + +def update_debugging(): + tool_to_platforms = {} + tool_to_boards = {} + vendors = {} + platforms = [] + frameworks = [] + for data in PlatformPackageManager().get_installed_boards(): + if not data.get("debug"): + continue + + for tool in data["debug"]["tools"]: + tool = str(tool) + if tool not in tool_to_platforms: + tool_to_platforms[tool] = [] + tool_to_platforms[tool].append(data["platform"]) + if tool not in tool_to_boards: + tool_to_boards[tool] = [] + tool_to_boards[tool].append(data["id"]) + + platforms.append(data["platform"]) + frameworks.extend(data["frameworks"]) + vendor = data["vendor"] + if vendor in vendors: + vendors[vendor].append(data) + else: + vendors[vendor] = [data] + + platforms = sorted(set(platforms)) + frameworks = sorted(set(frameworks)) + + lines = [".. _debugging_platforms:"] + lines.extend(generate_platforms_contents(platforms)) + lines.extend(generate_frameworks_contents(frameworks)) + + # Boards + lines.append( + """ +Boards +------ + +.. note:: + For more detailed ``board`` information please scroll tables below by horizontal. +""" + ) + for vendor, boards in sorted(vendors.items()): + lines.append(str(vendor)) + lines.append("~" * len(vendor)) + lines.extend(generate_boards_table(boards)) + + # save + with open( + os.path.join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+" + ) as fp: + content = fp.read() + fp.seek(0) + fp.truncate() + fp.write( + content[: content.index(".. _debugging_platforms:")] + "\n".join(lines) + ) + + # Debug tools + for tool, platforms in tool_to_platforms.items(): + tool_path = os.path.join(DOCS_ROOT_DIR, "plus", "debug-tools", "%s.rst" % tool) + if not os.path.isfile(tool_path): + if tool in SKIP_DEBUG_TOOLS: + click.secho("Skipped debug tool `%s`" % tool, fg="yellow") + else: + click.secho("Unknown debug tool `%s`" % tool, fg="red") + continue + platforms = sorted(set(platforms)) + + lines = [".. begin_platforms"] + lines.extend(generate_platforms_contents(platforms)) + tool_frameworks = [] + for platform in platforms: + for framework in frameworks: + if is_compat_platform_and_framework(platform, framework): + tool_frameworks.append(framework) + lines.extend(generate_frameworks_contents(tool_frameworks)) + + lines.append( + """ +Boards +------ + +.. note:: + For more detailed ``board`` information please scroll tables below by horizontal. +""" + ) + lines.extend( + generate_boards_table( + [ + b + for b in PlatformPackageManager().get_installed_boards() + if b["id"] in tool_to_boards[tool] + ], + skip_columns=None, + ) + ) + + with open(tool_path, "r+") as fp: + content = fp.read() + fp.seek(0) + fp.truncate() + fp.write(content[: content.index(".. begin_platforms")] + "\n".join(lines)) + + +def update_project_examples(): + platform_readme_tpl = """ +# {title}: development platform for [PlatformIO](https://platformio.org) + +{description} + +* [Home](https://platformio.org/platforms/{name}) (home page in PlatformIO Registry) +* [Documentation](https://docs.platformio.org/page/platforms/{name}.html) (advanced usage, packages, boards, frameworks, etc.) + +# Examples + +{examples} +""" + framework_readme_tpl = """ +# {title}: framework for [PlatformIO](https://platformio.org) + +{description} + +* [Home](https://platformio.org/frameworks/{name}) (home page in PlatformIO Registry) +* [Documentation](https://docs.platformio.org/page/frameworks/{name}.html) + +# Examples + +{examples} +""" + + project_examples_dir = os.path.join(fs.get_source_dir(), "..", "examples") + framework_examples_md_lines = {} + embedded = [] + desktop = [] + + for pkg in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(pkg) + github_url = p.repository_url[:-4] + + # Platform README + platform_examples_dir = os.path.join(p.get_dir(), "examples") + examples_md_lines = [] + if os.path.isdir(platform_examples_dir): + for item in sorted(os.listdir(platform_examples_dir)): + example_dir = os.path.join(platform_examples_dir, item) + if not os.path.isdir(example_dir) or not os.listdir(example_dir): + continue + url = "%s/tree/master/examples/%s" % (github_url, item) + examples_md_lines.append("* [%s](%s)" % (item, url)) + + readme_dir = os.path.join(project_examples_dir, "platforms", p.name) + if not os.path.isdir(readme_dir): + os.makedirs(readme_dir) + with open(os.path.join(readme_dir, "README.md"), "w") as fp: + fp.write( + platform_readme_tpl.format( + name=p.name, + title=p.title, + description=p.description, + examples="\n".join(examples_md_lines), + ) + ) + + # Framework README + for framework in get_frameworks(): + if not is_compat_platform_and_framework(p.name, framework["name"]): + continue + if framework["name"] not in framework_examples_md_lines: + framework_examples_md_lines[framework["name"]] = [] + lines = [] + lines.append("- [%s](%s)" % (p.title, github_url)) + lines.extend(" %s" % line for line in examples_md_lines) + lines.append("") + framework_examples_md_lines[framework["name"]].extend(lines) + + # Root README + line = "* [%s](%s)" % (p.title, "%s/tree/master/examples" % github_url) + if p.is_embedded(): + embedded.append(line) + else: + desktop.append(line) + + # Frameworks + frameworks = [] + for framework in get_frameworks(): + if framework["name"] not in framework_examples_md_lines: + continue + readme_dir = os.path.join(project_examples_dir, "frameworks", framework["name"]) + if not os.path.isdir(readme_dir): + os.makedirs(readme_dir) + with open(os.path.join(readme_dir, "README.md"), "w") as fp: + fp.write( + framework_readme_tpl.format( + name=framework["name"], + title=framework["title"], + description=framework["description"], + examples="\n".join(framework_examples_md_lines[framework["name"]]), + ) + ) + url = campaign_url( + "https://docs.platformio.org/en/latest/frameworks/%s.html#examples" + % framework["name"], + source="github", + medium="examples", + ) + frameworks.append("* [%s](%s)" % (framework["title"], url)) + + with open(os.path.join(project_examples_dir, "README.md"), "w") as fp: + fp.write( + """# PlatformIO Project Examples + +- [Development platforms](#development-platforms): + - [Embedded](#embedded) + - [Desktop](#desktop) +- [Frameworks](#frameworks) + +## Development platforms + +### Embedded + +%s + +### Desktop + +%s + +## Frameworks + +%s +""" + % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks)) + ) + + +def main(): + with tempfile.TemporaryDirectory() as tmp_dir: + print("Core directory: %s" % tmp_dir) + os.environ["PLATFORMIO_CORE_DIR"] = tmp_dir + install_platforms() + update_platform_docs() + update_framework_docs() + update_boards() + update_debugging() + update_project_examples() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fixsymlink.py b/scripts/fixsymlink.py new file mode 100644 index 0000000..0f9daa2 --- /dev/null +++ b/scripts/fixsymlink.py @@ -0,0 +1,40 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import chdir, getcwd, readlink, remove, symlink, walk +from os.path import exists, islink, join, relpath +from sys import exit as sys_exit + + +def fix_symlink(root, fname, brokenlink): + print(root, fname, brokenlink) + prevcwd = getcwd() + + chdir(root) + remove(fname) + symlink(relpath(brokenlink, root), fname) + chdir(prevcwd) + + +def main(): + for root, dirnames, filenames in walk("."): + for f in filenames: + path = join(root, f) + if not islink(path) or exists(path): + continue + fix_symlink(root, f, readlink(path)) + + +if __name__ == "__main__": + sys_exit(main()) diff --git a/scripts/install_devplatforms.py b/scripts/install_devplatforms.py new file mode 100644 index 0000000..20481ab --- /dev/null +++ b/scripts/install_devplatforms.py @@ -0,0 +1,62 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import subprocess +import sys + +import click + + +@click.command() +@click.option("--desktop", is_flag=True, default=False) +@click.option( + "--names", + envvar="PIO_INSTALL_DEVPLATFORM_NAMES", + help="Install specified platform (split by comma)", +) +@click.option( + "--ownernames", + envvar="PIO_INSTALL_DEVPLATFORM_OWNERNAMES", + help="Filter by ownernames (split by comma)", +) +def main(desktop, names, ownernames): + platforms = json.loads( + subprocess.check_output(["pio", "platform", "search", "--json-output"]).decode() + ) + names = [n.strip() for n in (names or "").split(",") if n.strip()] + ownernames = [n.strip() for n in (ownernames or "").split(",") if n.strip()] + for platform in platforms: + skip = [ + not desktop and platform["forDesktop"], + names and platform["name"] not in names, + ownernames and platform["ownername"] not in ownernames, + ] + if any(skip): + continue + subprocess.check_call( + [ + "pio", + "pkg", + "install", + "--global", + "--skip-dependencies", + "--platform", + "{ownername}/{name}".format(**platform), + ] + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..46716cc --- /dev/null +++ b/setup.py @@ -0,0 +1,96 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import platform +from setuptools import find_packages, setup + +from platformio import ( + __author__, + __description__, + __email__, + __license__, + __title__, + __url__, + __version__, + __install_requires__, +) + +# issue #4702; Broken "requests/charset_normalizer" on macOS ARM +if platform.system() == "Darwin" and "arm" in platform.machine().lower(): + __install_requires__.append("chardet>=3.0.2,<4") + + +setup( + name=__title__, + version=__version__, + description=__description__, + long_description=open("README.rst").read(), + author=__author__, + author_email=__email__, + url=__url__, + license=__license__, + install_requires=__install_requires__, + python_requires=">=3.6", + packages=find_packages(include=["platformio", "platformio.*"]), + package_data={ + "platformio": [ + "assets/system/99-platformio-udev.rules", + "project/integration/tpls/*/*.tpl", + "project/integration/tpls/*/.*.tpl", # include hidden files + "project/integration/tpls/*/.*/*.tpl", # include hidden folders + "project/integration/tpls/*/*/*.tpl", # NetBeans + "project/integration/tpls/*/*/*/*.tpl", # NetBeans + ] + }, + entry_points={ + "console_scripts": [ + "platformio = platformio.__main__:main", + "pio = platformio.__main__:main", + "piodebuggdb = platformio.__main__:debug_gdb_main", + ] + }, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: C", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Topic :: Software Development", + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Compilers", + ], + keywords=[ + "iot", + "embedded", + "arduino", + "mbed", + "esp8266", + "esp32", + "fpga", + "firmware", + "continuous-integration", + "cloud-ide", + "avr", + "arm", + "ide", + "unit-testing", + "hardware", + "verilog", + "microcontroller", + "debug", + ], +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/tests/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/pkg/__init__.py b/tests/commands/pkg/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/tests/commands/pkg/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/pkg/test_exec.py b/tests/commands/pkg/test_exec.py new file mode 100644 index 0000000..0b0c0f4 --- /dev/null +++ b/tests/commands/pkg/test_exec.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import pytest + +from platformio.package.commands.exec import package_exec_cmd +from platformio.util import strip_ansi_codes + + +def test_pkg_not_installed(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + package_exec_cmd, + ["--", "openocd"], + ) + with pytest.raises( + AssertionError, + match=("Could not find a package with 'openocd' executable file"), + ): + validate_cliresult(result) + + +def test_pkg_specified(clirunner, validate_cliresult, isolated_pio_core): + # with install + result = clirunner.invoke( + package_exec_cmd, + ["-p", "platformio/tool-openocd", "--", "openocd", "--version"], + obj=dict(force_click_stream=True), + ) + validate_cliresult(result) + output = strip_ansi_codes(result.output) + assert "Tool Manager: Installing platformio/tool-openocd" in output + assert "Open On-Chip Debugger" in output + + +def test_unrecognized_options(clirunner, validate_cliresult, isolated_pio_core): + # unrecognized option + result = clirunner.invoke( + package_exec_cmd, + ["--", "openocd", "--test-unrecognized"], + obj=dict(force_click_stream=True), + ) + with pytest.raises( + AssertionError, + match=(r"openocd: (unrecognized|unknown) option"), + ): + validate_cliresult(result) diff --git a/tests/commands/pkg/test_install.py b/tests/commands/pkg/test_install.py new file mode 100644 index 0000000..edcbc24 --- /dev/null +++ b/tests/commands/pkg/test_install.py @@ -0,0 +1,592 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import os + +import pytest + +from platformio import __core_packages__, fs +from platformio.package.commands.install import package_install_cmd +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.project.config import ProjectConfig + +PROJECT_CONFIG_TPL = """ +[env] +platform = platformio/atmelavr@^3.4.0 +lib_deps = + milesburton/DallasTemperature@^3.9.1 + https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip + +[env:baremetal] +board = uno + +[env:devkit] +framework = arduino +board = attiny88 +""" + + +def pkgs_to_specs(pkgs): + return [ + PackageSpec(name=pkg.metadata.name, requirements=pkg.metadata.version) + for pkg in pkgs + ] + + +def test_global_packages( + clirunner, + validate_cliresult, + func_isolated_pio_core, + get_pkg_latest_version, + tmp_path, +): + # libraries + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "-l", + "https://github.com/milesburton/Arduino-Temperature-Control-Library.git#3.9.0", + "--skip-dependencies", + ], + ) + validate_cliresult(result) + assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ + PackageSpec("DallasTemperature@3.9.0+sha.964939d") + ] + # with dependencies + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "-l", + "https://github.com/milesburton/Arduino-Temperature-Control-Library.git#3.9.0", + "-l", + "bblanchon/ArduinoJson@^5", + ], + ) + validate_cliresult(result) + assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ + PackageSpec("ArduinoJson@5.13.4"), + PackageSpec("DallasTemperature@3.9.0+sha.964939d"), + PackageSpec("OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire")), + ] + # custom storage + storage_dir = tmp_path / "custom_lib_storage" + storage_dir.mkdir() + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "--storage-dir", + str(storage_dir), + "-l", + "bblanchon/ArduinoJson@^5", + ], + ) + validate_cliresult(result) + assert pkgs_to_specs(LibraryPackageManager(storage_dir).get_installed()) == [ + PackageSpec("ArduinoJson@5.13.4") + ] + + # tools + result = clirunner.invoke( + package_install_cmd, + ["--global", "-t", "platformio/framework-arduino-avr-attiny@^1.5.2"], + ) + validate_cliresult(result) + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("framework-arduino-avr-attiny@1.5.2") + ] + + # platforms + result = clirunner.invoke( + package_install_cmd, + ["--global", "-p", "platformio/atmelavr@^3.4.0", "--skip-dependencies"], + ) + validate_cliresult(result) + assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ + PackageSpec("atmelavr@3.4.0") + ] + + +def test_skip_dependencies( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "--skip-dependencies"], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + installed_lib_pkgs = LibraryPackageManager( + os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "devkit") + ).get_installed() + assert pkgs_to_specs(installed_lib_pkgs) == [ + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + ] + assert len(ToolPackageManager().get_installed()) == 1 # SCons + + +def test_baremetal_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "baremetal"], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + installed_lib_pkgs = LibraryPackageManager( + os.path.join(ProjectConfig().get("platformio", "libdeps_dir"), "baremetal") + ).get_installed() + assert pkgs_to_specs(installed_lib_pkgs) == [ + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + ] + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("tool-scons@%s" % __core_packages__["tool-scons"][1:]), + PackageSpec("toolchain-atmelavr@1.70300.191015"), + ] + + +def test_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_specs(lm.get_installed()) == [ + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + ] + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("framework-arduino-avr-attiny@1.5.2"), + PackageSpec("tool-scons@%s" % __core_packages__["tool-scons"][1:]), + PackageSpec("toolchain-atmelavr@1.70300.191015"), + ] + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.9.1", + "https://github.com/esphome/ESPAsyncWebServer/archive/refs/tags/v2.1.0.zip", + ] + + # test "Already up-to-date" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + assert "Already up-to-date" in result.output + + +def test_private_lib_deps( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): + project_dir = tmp_path / "project" + private_lib_dir = project_dir / "lib" / "private" + private_lib_dir.mkdir(parents=True) + (private_lib_dir / "library.json").write_text( + """ +{ + "name": "My Private Lib", + "version": "1.0.0", + "dependencies": { + "bblanchon/ArduinoJson": "^5", + "milesburton/DallasTemperature": "^3.9.1" + } +} +""" + ) + (project_dir / "platformio.ini").write_text( + """ +[env:private] +platform = native +""" + ) + with fs.cd(str(project_dir)): + config = ProjectConfig() + + # some deps were added by user manually + result = clirunner.invoke( + package_install_cmd, + [ + "-g", + "--storage-dir", + config.get("platformio", "lib_dir"), + "-l", + "paulstoffregen/OneWire@^2.3.5", + ], + ) + validate_cliresult(result) + + # ensure all deps are installed + result = clirunner.invoke(package_install_cmd) + validate_cliresult(result) + installed_private_pkgs = LibraryPackageManager( + config.get("platformio", "lib_dir") + ).get_installed() + assert pkgs_to_specs(installed_private_pkgs) == [ + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + PackageSpec("My Private Lib@1.0.0"), + ] + installed_env_pkgs = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "private") + ).get_installed() + assert pkgs_to_specs(installed_env_pkgs) == [ + PackageSpec("ArduinoJson@5.13.4"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + ] + + +def test_remove_project_unused_libdeps( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "baremetal"], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + config = ProjectConfig() + storage_dir = os.path.join(config.get("platformio", "libdeps_dir"), "baremetal") + lm = LibraryPackageManager(storage_dir) + assert pkgs_to_specs(lm.get_installed()) == [ + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + ] + + # add new deps + lib_deps = config.get("env:baremetal", "lib_deps") + config.set("env:baremetal", "lib_deps", lib_deps + ["bblanchon/ArduinoJson@^5"]) + config.save() + result = clirunner.invoke( + package_install_cmd, + ["-e", "baremetal"], + ) + validate_cliresult(result) + lm = LibraryPackageManager(storage_dir) + assert pkgs_to_specs(lm.get_installed()) == [ + PackageSpec("ArduinoJson@5.13.4"), + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec("ESPAsyncWebServer-esphome@2.1.0"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + ] + + # manually remove from cofiguration file + config.set("env:baremetal", "lib_deps", ["bblanchon/ArduinoJson@^5"]) + config.save() + result = clirunner.invoke( + package_install_cmd, + ["-e", "baremetal"], + ) + validate_cliresult(result) + lm = LibraryPackageManager(storage_dir) + assert pkgs_to_specs(lm.get_installed()) == [PackageSpec("ArduinoJson@5.13.4")] + + +def test_unknown_project_dependencies( + clirunner, validate_cliresult, isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text( + """ +[env:unknown_platform] +platform = unknown_platform + +[env:unknown_lib_deps] +lib_deps = SPI, platformio/unknown_library +""" + ) + with fs.cd(str(project_dir)): + result = clirunner.invoke( + package_install_cmd, + ["-e", "unknown_platform"], + ) + with pytest.raises( + AssertionError, + match=("Could not find the package with 'unknown_platform' requirements"), + ): + validate_cliresult(result) + + # unknown libraries + result = clirunner.invoke( + package_install_cmd, + ["-e", "unknown_lib_deps"], + ) + with pytest.raises( + AssertionError, + match=( + "Could not find the package with 'platformio/unknown_library' requirements" + ), + ): + validate_cliresult(result) + + +def test_custom_project_libraries( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "bblanchon/ArduinoJson@^5" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-l", spec], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + # try again + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-l", spec], + ) + validate_cliresult(result) + assert "already installed" in result.output + # try again in the silent mode + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-l", spec, "--silent"], + ) + validate_cliresult(result) + assert not result.output.strip() + + # check folders + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_specs(lm.get_installed()) == [PackageSpec("ArduinoJson@5.13.4")] + # do not expect any platforms/tools + assert not os.path.exists(config.get("platformio", "platforms_dir")) + assert not os.path.exists(config.get("platformio", "packages_dir")) + + # check saved deps + assert config.get("env:devkit", "lib_deps") == [ + "bblanchon/ArduinoJson@^5", + ] + + # install library without saving to config + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-l", "nanopb/Nanopb@^0.4.6", "--no-save"], + ) + validate_cliresult(result) + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_specs(lm.get_installed()) == [ + PackageSpec("ArduinoJson@5.13.4"), + PackageSpec("Nanopb@0.4.7"), + ] + assert config.get("env:devkit", "lib_deps") == [ + "bblanchon/ArduinoJson@^5", + ] + + # unknown libraries + result = clirunner.invoke( + package_install_cmd, ["-l", "platformio/unknown_library"] + ) + with pytest.raises( + AssertionError, + match=( + "Could not find the package with " + "'platformio/unknown_library' requirements" + ), + ): + validate_cliresult(result) + + +def test_custom_project_tools( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "platformio/tool-openocd @ ^2" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-t", spec], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + # try again + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-t", spec], + ) + validate_cliresult(result) + assert "already installed" in result.output + # try again in the silent mode + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-t", spec, "--silent"], + ) + validate_cliresult(result) + assert not result.output.strip() + + config = ProjectConfig() + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("tool-openocd@2.1100.211028") + ] + assert not LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ).get_installed() + # do not expect any platforms + assert not os.path.exists(config.get("platformio", "platforms_dir")) + + # check saved deps + assert config.get("env:devkit", "platform_packages") == [ + "platformio/tool-openocd@^2", + ] + + # install tool without saving to config + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-t", "platformio/tool-esptoolpy@1.20310.0", "--no-save"], + ) + validate_cliresult(result) + config = ProjectConfig() + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("tool-esptoolpy@1.20310.0"), + PackageSpec("tool-openocd@2.1100.211028"), + ] + assert config.get("env:devkit", "platform_packages") == [ + "platformio/tool-openocd@^2", + ] + + # unknown tool + result = clirunner.invoke( + package_install_cmd, ["-t", "platformio/unknown_tool"] + ) + with pytest.raises( + AssertionError, + match=( + "Could not find the package with " + "'platformio/unknown_tool' requirements" + ), + ): + validate_cliresult(result) + + +def test_custom_project_platforms( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "atmelavr@^3.4.0" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-p", spec, "--skip-dependencies"], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + # try again + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-p", spec, "--skip-dependencies"], + ) + validate_cliresult(result) + assert "already installed" in result.output + # try again in the silent mode + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-p", spec, "--silent", "--skip-dependencies"], + ) + validate_cliresult(result) + assert not result.output.strip() + + config = ProjectConfig() + assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ + PackageSpec("atmelavr@3.4.0") + ] + assert not LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ).get_installed() + # do not expect any packages + assert not os.path.exists(config.get("platformio", "packages_dir")) + + # unknown platform + result = clirunner.invoke(package_install_cmd, ["-p", "unknown_platform"]) + with pytest.raises( + AssertionError, + match="Could not find the package with 'unknown_platform' requirements", + ): + validate_cliresult(result) + + # incompatible board + result = clirunner.invoke(package_install_cmd, ["-e", "devkit", "-p", "sifive"]) + with pytest.raises( + AssertionError, + match="Unknown board ID", + ): + validate_cliresult(result) diff --git a/tests/commands/pkg/test_list.py b/tests/commands/pkg/test_list.py new file mode 100644 index 0000000..c917879 --- /dev/null +++ b/tests/commands/pkg/test_list.py @@ -0,0 +1,137 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.list import package_list_cmd + +PROJECT_CONFIG_TPL = """ +[env] +platform = platformio/atmelavr@^3.4.0 + +[env:baremetal] +board = uno + +[env:devkit] +framework = arduino +board = attiny88 +lib_deps = + milesburton/DallasTemperature@^3.9.1 + https://github.com/bblanchon/ArduinoJson.git#v6.19.0 +""" + + +def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + + # test all envs + result = clirunner.invoke( + package_list_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + assert all(token in result.output for token in ("baremetal", "devkit")) + assert result.output.count("Platform atmelavr @ 3.4.0") == 2 + assert ( + result.output.count( + "toolchain-atmelavr @ 1.70300.191015 (required: " + "platformio/toolchain-atmelavr @ ~1.70300.0)" + ) + == 2 + ) + assert result.output.count("Libraries") == 1 + assert ( + "ArduinoJson @ 6.19.0+sha.9693fd2 (required: " + "git+https://github.com/bblanchon/ArduinoJson.git#v6.19.0)" + ) in result.output + assert "OneWire @ 2" in result.output + + # test "baremetal" + result = clirunner.invoke( + package_list_cmd, + ["-d", str(project_dir), "-e", "baremetal"], + ) + validate_cliresult(result) + assert "Platform atmelavr @ 3" in result.output + assert "Libraries" not in result.output + + # filter by "tool" package + result = clirunner.invoke( + package_list_cmd, + ["-d", str(project_dir), "-t", "toolchain-atmelavr@~1.70300.0"], + ) + assert "framework-arduino" not in result.output + assert "Libraries" not in result.output + + # list only libraries + result = clirunner.invoke( + package_list_cmd, + ["-d", str(project_dir), "--only-libraries"], + ) + assert "Platform atmelavr" not in result.output + + # list only libraries for baremetal + result = clirunner.invoke( + package_list_cmd, + ["-d", str(project_dir), "-e", "baremetal", "--only-libraries"], + ) + assert "No packages" in result.output + + +def test_global_packages(clirunner, validate_cliresult, isolated_pio_core, tmp_path): + result = clirunner.invoke(package_list_cmd, ["-g"]) + validate_cliresult(result) + assert "atmelavr @ 3" in result.output + assert "framework-arduino-avr-attiny" in result.output + + # only tools + result = clirunner.invoke(package_list_cmd, ["-g", "--only-tools"]) + validate_cliresult(result) + assert "toolchain-atmelavr" in result.output + assert "Platforms" not in result.output + + # find tool package + result = clirunner.invoke(package_list_cmd, ["-g", "-t", "toolchain-atmelavr"]) + validate_cliresult(result) + assert "toolchain-atmelavr" in result.output + assert "framework-arduino-avr-attiny@" not in result.output + + # only libraries - no packages + result = clirunner.invoke(package_list_cmd, ["-g", "--only-libraries"]) + validate_cliresult(result) + assert not result.output.strip() + + # check global libs + result = clirunner.invoke( + package_install_cmd, ["-g", "-l", "milesburton/DallasTemperature@^3.9.1"] + ) + validate_cliresult(result) + result = clirunner.invoke(package_list_cmd, ["-g", "--only-libraries"]) + validate_cliresult(result) + assert "DallasTemperature" in result.output + assert "OneWire" in result.output + + # filter by lib + result = clirunner.invoke(package_list_cmd, ["-g", "-l", "OneWire"]) + validate_cliresult(result) + assert "DallasTemperature" in result.output + assert "OneWire" in result.output diff --git a/tests/commands/pkg/test_outdated.py b/tests/commands/pkg/test_outdated.py new file mode 100644 index 0000000..91180b4 --- /dev/null +++ b/tests/commands/pkg/test_outdated.py @@ -0,0 +1,62 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import re + +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.outdated import package_outdated_cmd + +PROJECT_OUTDATED_CONFIG_TPL = """ +[env:devkit] +platform = platformio/atmelavr@^2 +framework = arduino +board = attiny88 +lib_deps = milesburton/DallasTemperature@~3.8.0 +""" + +PROJECT_UPDATED_CONFIG_TPL = """ +[env:devkit] +platform = platformio/atmelavr@<4 +framework = arduino +board = attiny88 +lib_deps = milesburton/DallasTemperature@^3.8.0 +""" + + +def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) + result = clirunner.invoke(package_install_cmd, ["-d", str(project_dir)]) + validate_cliresult(result) + + # overwrite config + (project_dir / "platformio.ini").write_text(PROJECT_UPDATED_CONFIG_TPL) + result = clirunner.invoke(package_outdated_cmd, ["-d", str(project_dir)]) + validate_cliresult(result) + + # validate output + assert "Checking" in result.output + assert re.search( + r"^atmelavr\s+2\.2\.0\s+3\.\d+\.\d+\s+[456789]\.\d+\.\d+\s+Platform\s+devkit", + result.output, + re.MULTILINE, + ) + assert re.search( + r"^DallasTemperature\s+3\.8\.1\s+3\.\d+\.\d+\s+3\.\d+\.\d+\s+Library\s+devkit", + result.output, + re.MULTILINE, + ) diff --git a/tests/commands/pkg/test_search.py b/tests/commands/pkg/test_search.py new file mode 100644 index 0000000..284f436 --- /dev/null +++ b/tests/commands/pkg/test_search.py @@ -0,0 +1,58 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.commands.search import package_search_cmd + + +def test_empty_query(clirunner, validate_cliresult): + result = clirunner.invoke( + package_search_cmd, + [""], + ) + validate_cliresult(result) + assert all(t in result.output for t in ("Found", "Official", "page 1 of")) + + +def test_pagination(clirunner, validate_cliresult): + result = clirunner.invoke( + package_search_cmd, + ["type:tool"], + ) + validate_cliresult(result) + assert all(t in result.output for t in ("Verified Tool", "page 1 of")) + + result = clirunner.invoke( + package_search_cmd, + ["type:tool", "-p", "10"], + ) + validate_cliresult(result) + assert all(t in result.output for t in ("Tool", "page 10 of")) + + +def test_sorting(clirunner, validate_cliresult): + result = clirunner.invoke( + package_search_cmd, + ["OneWire", "-s", "popularity"], + ) + validate_cliresult(result) + assert "paulstoffregen/OneWire" in result.output + + +def test_not_found(clirunner, validate_cliresult): + result = clirunner.invoke( + package_search_cmd, + ["name:unknown-package"], + ) + validate_cliresult(result) + assert "Nothing has been found" in result.output diff --git a/tests/commands/pkg/test_show.py b/tests/commands/pkg/test_show.py new file mode 100644 index 0000000..979edbb --- /dev/null +++ b/tests/commands/pkg/test_show.py @@ -0,0 +1,103 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from platformio.exception import UserSideException +from platformio.package.commands.show import package_show_cmd + + +def test_spec_name(clirunner, validate_cliresult): + # library + result = clirunner.invoke( + package_show_cmd, + ["ArduinoJSON"], + ) + validate_cliresult(result) + assert "bblanchon/ArduinoJson" in result.output + assert "Library" in result.output + + # platform + result = clirunner.invoke( + package_show_cmd, + ["espressif32"], + ) + validate_cliresult(result) + assert "platformio/espressif32" in result.output + assert "Platform" in result.output + + # tool + result = clirunner.invoke( + package_show_cmd, + ["tool-jlink"], + ) + validate_cliresult(result) + assert "platformio/tool-jlink" in result.output + assert "tool" in result.output + + +def test_spec_owner(clirunner, validate_cliresult): + result = clirunner.invoke( + package_show_cmd, + ["bblanchon/ArduinoJSON"], + ) + validate_cliresult(result) + assert "bblanchon/ArduinoJson" in result.output + assert "Library" in result.output + + # test broken owner + result = clirunner.invoke( + package_show_cmd, + ["unknown/espressif32"], + ) + with pytest.raises(UserSideException, match="Could not find"): + raise result.exception + + +def test_complete_spec(clirunner, validate_cliresult): + result = clirunner.invoke( + package_show_cmd, + ["bblanchon/ArduinoJSON", "-t", "library"], + ) + validate_cliresult(result) + assert "bblanchon/ArduinoJson" in result.output + assert "Library" in result.output + + # tool + result = clirunner.invoke( + package_show_cmd, + ["platformio/tool-jlink", "-t", "tool"], + ) + validate_cliresult(result) + assert "platformio/tool-jlink" in result.output + assert "tool" in result.output + + +def test_name_conflict(clirunner): + result = clirunner.invoke( + package_show_cmd, + ["OneWire", "-t", "library"], + ) + assert "More than one package" in result.output + assert isinstance(result.exception, UserSideException) + + +def test_spec_version(clirunner, validate_cliresult): + result = clirunner.invoke( + package_show_cmd, + ["bblanchon/ArduinoJSON@5.13.4"], + ) + validate_cliresult(result) + assert "bblanchon/ArduinoJson" in result.output + assert "Library • 5.13.4" in result.output diff --git a/tests/commands/pkg/test_uninstall.py b/tests/commands/pkg/test_uninstall.py new file mode 100644 index 0000000..73e2179 --- /dev/null +++ b/tests/commands/pkg/test_uninstall.py @@ -0,0 +1,408 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import os + +from platformio import fs +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.uninstall import package_uninstall_cmd +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.project.config import ProjectConfig + +PROJECT_CONFIG_TPL = """ +[env] +platform = platformio/atmelavr@^3.4.0 +lib_deps = milesburton/DallasTemperature@^3.9.1 + +[env:baremetal] +board = uno + +[env:devkit] +framework = arduino +board = attiny88 +""" + + +def pkgs_to_names(pkgs): + return [pkg.metadata.name for pkg in pkgs] + + +def test_global_packages( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + # libraries + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "-l", + "marvinroger/Homie@^3.0.1", + ], + ) + validate_cliresult(result) + assert pkgs_to_names(LibraryPackageManager().get_installed()) == [ + "ArduinoJson", + "AsyncMqttClient", + "AsyncTCP", + "Bounce2", + "ESP Async WebServer", + "ESPAsyncTCP", + "Homie", + ] + # uninstall all deps + result = clirunner.invoke( + package_uninstall_cmd, + [ + "--global", + "-l", + "Homie", + ], + ) + validate_cliresult(result) + assert not pkgs_to_names(LibraryPackageManager().get_installed()) + + # skip dependencies + validate_cliresult( + clirunner.invoke( + package_install_cmd, + [ + "--global", + "-l", + "marvinroger/Homie@^3.0.1", + ], + ) + ) + result = clirunner.invoke( + package_uninstall_cmd, + ["--global", "-l", "marvinroger/Homie@^3.0.1", "--skip-dependencies"], + ) + validate_cliresult(result) + assert pkgs_to_names(LibraryPackageManager().get_installed()) == [ + "ArduinoJson", + "AsyncMqttClient", + "AsyncTCP", + "Bounce2", + "ESP Async WebServer", + "ESPAsyncTCP", + ] + # remove specific dependency + result = clirunner.invoke( + package_uninstall_cmd, + [ + "--global", + "-l", + "ESP Async WebServer", + ], + ) + validate_cliresult(result) + assert pkgs_to_names(LibraryPackageManager().get_installed()) == [ + "ArduinoJson", + "AsyncMqttClient", + "Bounce2", + ] + + # custom storage + storage_dir = tmp_path / "custom_lib_storage" + storage_dir.mkdir() + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "--storage-dir", + str(storage_dir), + "-l", + "marvinroger/Homie@^3.0.1", + "--skip-dependencies", + ], + ) + validate_cliresult(result) + assert pkgs_to_names(LibraryPackageManager(storage_dir).get_installed()) == [ + "Homie" + ] + result = clirunner.invoke( + package_uninstall_cmd, + [ + "--global", + "--storage-dir", + str(storage_dir), + "-l", + "marvinroger/Homie@^3.0.1", + ], + ) + validate_cliresult(result) + assert not pkgs_to_names(LibraryPackageManager(storage_dir).get_installed()) + + # tools + result = clirunner.invoke( + package_install_cmd, + ["--global", "-t", "platformio/framework-arduino-avr-attiny@^1.5.2"], + ) + validate_cliresult(result) + assert pkgs_to_names(ToolPackageManager().get_installed()) == [ + "framework-arduino-avr-attiny" + ] + result = clirunner.invoke( + package_uninstall_cmd, + ["--global", "-t", "framework-arduino-avr-attiny"], + ) + validate_cliresult(result) + assert not pkgs_to_names(ToolPackageManager().get_installed()) + + # platforms + result = clirunner.invoke( + package_install_cmd, + ["--global", "-p", "platformio/atmelavr@^3.4.0"], + ) + validate_cliresult(result) + assert pkgs_to_names(PlatformPackageManager().get_installed()) == ["atmelavr"] + assert pkgs_to_names(ToolPackageManager().get_installed()) == ["toolchain-atmelavr"] + result = clirunner.invoke( + package_uninstall_cmd, + ["--global", "-p", "platformio/atmelavr@^3.4.0"], + ) + validate_cliresult(result) + assert not pkgs_to_names(PlatformPackageManager().get_installed()) + assert not pkgs_to_names(ToolPackageManager().get_installed()) + + +def test_project(clirunner, validate_cliresult, isolated_pio_core, tmp_path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_names(lm.get_installed()) == ["DallasTemperature", "OneWire"] + assert pkgs_to_names(ToolPackageManager().get_installed()) == [ + "framework-arduino-avr-attiny", + "tool-scons", + "toolchain-atmelavr", + ] + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.9.1" + ] + + # try again + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + assert "Already up-to-date" in result.output + + # uninstall + result = clirunner.invoke( + package_uninstall_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert not pkgs_to_names(lm.get_installed()) + assert pkgs_to_names(ToolPackageManager().get_installed()) == ["tool-scons"] + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.9.1" + ] + + +def test_custom_project_libraries( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "bblanchon/ArduinoJson@^6.19.2" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-l", spec], + ) + validate_cliresult(result) + assert "Already up-to-date" not in result.output + with fs.cd(str(project_dir)): + # check folders + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_names(lm.get_installed()) == ["ArduinoJson"] + # do not expect any platforms/tools + assert not os.path.exists(config.get("platformio", "platforms_dir")) + assert not os.path.exists(config.get("platformio", "packages_dir")) + # check saved deps + assert config.get("env:devkit", "lib_deps") == [ + "bblanchon/ArduinoJson@^6.19.2", + ] + # uninstall + result = clirunner.invoke( + package_uninstall_cmd, + ["-e", "devkit", "-l", spec], + ) + validate_cliresult(result) + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert not pkgs_to_names(lm.get_installed()) + # do not expect any platforms/tools + assert not os.path.exists(config.get("platformio", "platforms_dir")) + assert not os.path.exists(config.get("platformio", "packages_dir")) + # check saved deps + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.9.1" + ] + + # install library without saving to config + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-l", spec, "--no-save"], + ) + validate_cliresult(result) + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_names(lm.get_installed()) == ["ArduinoJson"] + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.9.1", + ] + result = clirunner.invoke( + package_uninstall_cmd, + ["-e", "devkit", "-l", spec, "--no-save"], + ) + validate_cliresult(result) + config = ProjectConfig() + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.9.1", + ] + + # unknown libraries + result = clirunner.invoke( + package_uninstall_cmd, ["-l", "platformio/unknown_library"] + ) + assert isinstance(result.exception, UnknownPackageError) + + +def test_custom_project_tools( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "platformio/tool-openocd@^2" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-t", spec], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + config = ProjectConfig() + assert pkgs_to_names(ToolPackageManager().get_installed()) == ["tool-openocd"] + assert not LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ).get_installed() + # do not expect any platforms + assert not os.path.exists(config.get("platformio", "platforms_dir")) + # check saved deps + assert config.get("env:devkit", "platform_packages") == [ + "platformio/tool-openocd@^2", + ] + # uninstall + result = clirunner.invoke( + package_uninstall_cmd, + ["-e", "devkit", "-t", spec], + ) + validate_cliresult(result) + assert not pkgs_to_names(ToolPackageManager().get_installed()) + # check saved deps + assert not ProjectConfig().get("env:devkit", "platform_packages") + + # install tool without saving to config + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-t", "platformio/tool-esptoolpy@1.20310.0"], + ) + validate_cliresult(result) + assert pkgs_to_names(ToolPackageManager().get_installed()) == [ + "tool-esptoolpy", + ] + assert ProjectConfig().get("env:devkit", "platform_packages") == [ + "platformio/tool-esptoolpy@1.20310.0", + ] + # uninstall + result = clirunner.invoke( + package_uninstall_cmd, + ["-e", "devkit", "-t", "platformio/tool-esptoolpy@^1", "--no-save"], + ) + validate_cliresult(result) + assert not pkgs_to_names(ToolPackageManager().get_installed()) + assert ProjectConfig().get("env:devkit", "platform_packages") == [ + "platformio/tool-esptoolpy@1.20310.0", + ] + + # unknown tool + result = clirunner.invoke( + package_uninstall_cmd, ["-t", "platformio/unknown_tool"] + ) + assert isinstance(result.exception, UnknownPackageError) + + +def test_custom_project_platforms( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + spec = "platformio/atmelavr@^3.4.0" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-p", spec], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + config = ProjectConfig() + assert pkgs_to_names(PlatformPackageManager().get_installed()) == ["atmelavr"] + assert not LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ).get_installed() + assert pkgs_to_names(ToolPackageManager().get_installed()) == [ + "framework-arduino-avr-attiny", + "toolchain-atmelavr", + ] + # uninstall + result = clirunner.invoke( + package_uninstall_cmd, + ["-e", "devkit", "-p", spec], + ) + validate_cliresult(result) + assert not pkgs_to_names(PlatformPackageManager().get_installed()) + assert not pkgs_to_names(ToolPackageManager().get_installed()) + + # unknown platform + result = clirunner.invoke(package_uninstall_cmd, ["-p", "unknown_platform"]) + assert isinstance(result.exception, UnknownPackageError) diff --git a/tests/commands/pkg/test_update.py b/tests/commands/pkg/test_update.py new file mode 100644 index 0000000..5656daa --- /dev/null +++ b/tests/commands/pkg/test_update.py @@ -0,0 +1,374 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import os + +from platformio import __core_packages__, fs +from platformio.package.commands.install import package_install_cmd +from platformio.package.commands.update import package_update_cmd +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.project.config import ProjectConfig + +PROJECT_OUTDATED_CONFIG_TPL = """ +[env:devkit] +platform = platformio/atmelavr@^2 +framework = arduino +board = attiny88 +lib_deps = milesburton/DallasTemperature@~3.8.0 +""" + +PROJECT_UPDATED_CONFIG_TPL = """ +[env:devkit] +platform = platformio/atmelavr@<4 +framework = arduino +board = attiny88 +lib_deps = milesburton/DallasTemperature@^3.8.0 +""" + + +def pkgs_to_specs(pkgs): + return [ + PackageSpec(name=pkg.metadata.name, requirements=pkg.metadata.version) + for pkg in pkgs + ] + + +def test_global_packages( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + # libraries + result = clirunner.invoke( + package_install_cmd, + ["--global", "-l", "bblanchon/ArduinoJson@^5"], + ) + validate_cliresult(result) + assert pkgs_to_specs(LibraryPackageManager().get_installed()) == [ + PackageSpec("ArduinoJson@5.13.4") + ] + # update to the latest version + result = clirunner.invoke( + package_update_cmd, + ["--global", "-l", "bblanchon/ArduinoJson"], + ) + validate_cliresult(result) + pkgs = LibraryPackageManager().get_installed() + assert len(pkgs) == 1 + assert pkgs[0].metadata.version.major > 5 + # custom storage + storage_dir = tmp_path / "custom_lib_storage" + storage_dir.mkdir() + result = clirunner.invoke( + package_install_cmd, + [ + "--global", + "--storage-dir", + str(storage_dir), + "-l", + "bblanchon/ArduinoJson@^5", + ], + ) + validate_cliresult(result) + assert pkgs_to_specs(LibraryPackageManager(storage_dir).get_installed()) == [ + PackageSpec("ArduinoJson@5.13.4") + ] + # update to the latest version + result = clirunner.invoke( + package_update_cmd, + ["--global", "--storage-dir", str(storage_dir), "-l", "bblanchon/ArduinoJson"], + ) + validate_cliresult(result) + pkgs = LibraryPackageManager(storage_dir).get_installed() + assert len(pkgs) == 1 + assert pkgs[0].metadata.version.major > 5 + + # tools + result = clirunner.invoke( + package_install_cmd, + ["--global", "-t", "platformio/framework-arduino-avr-attiny@~1.4"], + ) + validate_cliresult(result) + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("framework-arduino-avr-attiny@1.4.1") + ] + # update to the latest version + result = clirunner.invoke( + package_update_cmd, + ["--global", "-t", "platformio/framework-arduino-avr-attiny@^1"], + ) + validate_cliresult(result) + pkgs = ToolPackageManager().get_installed() + assert len(pkgs) == 1 + assert pkgs[0].metadata.version.major == 1 + assert pkgs[0].metadata.version.minor > 4 + + # platforms + result = clirunner.invoke( + package_install_cmd, + ["--global", "-p", "platformio/atmelavr@^2", "--skip-dependencies"], + ) + validate_cliresult(result) + assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ + PackageSpec("atmelavr@2.2.0") + ] + # update to the latest version + result = clirunner.invoke( + package_update_cmd, + ["--global", "-p", "platformio/atmelavr", "--skip-dependencies"], + ) + validate_cliresult(result) + pkgs = PlatformPackageManager().get_installed() + assert len(pkgs) == 1 + assert pkgs[0].metadata.version.major > 2 + + # update unknown package + result = clirunner.invoke( + package_update_cmd, + ["--global", "-l", "platformio/unknown_package_for_update"], + ) + assert isinstance(result.exception, UnknownPackageError) + + +def test_project( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir)], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_specs(lm.get_installed()) == [ + PackageSpec("DallasTemperature@3.8.1"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + ] + assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ + PackageSpec("atmelavr@2.2.0") + ] + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("framework-arduino-avr-attiny@1.3.2"), + PackageSpec("tool-scons@%s" % __core_packages__["tool-scons"][1:]), + PackageSpec("toolchain-atmelavr@1.50400.190710"), + ] + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@~3.8.0" + ] + + # update packages + (project_dir / "platformio.ini").write_text(PROJECT_UPDATED_CONFIG_TPL) + result = clirunner.invoke(package_update_cmd) + validate_cliresult(result) + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + pkgs = PlatformPackageManager().get_installed() + assert len(pkgs) == 1 + assert pkgs[0].metadata.name == "atmelavr" + assert pkgs[0].metadata.version.major == 3 + assert pkgs_to_specs(lm.get_installed()) == [ + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + ] + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("framework-arduino-avr-attiny@1.3.2"), + PackageSpec("tool-scons@%s" % __core_packages__["tool-scons"][1:]), + PackageSpec("toolchain-atmelavr@1.70300.191015"), + PackageSpec("toolchain-atmelavr@1.50400.190710"), + ] + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.8.0" + ] + + # update again + result = clirunner.invoke(package_update_cmd) + validate_cliresult(result) + assert "Already up-to-date." in result.output + + # update again in the silent ,pde + result = clirunner.invoke(package_update_cmd, ["--silent"]) + validate_cliresult(result) + assert not result.output + + +def test_custom_project_libraries( + clirunner, validate_cliresult, isolated_pio_core, get_pkg_latest_version, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) + spec = "milesburton/DallasTemperature@~3.8.0" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-l", spec], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + config = ProjectConfig() + assert config.get("env:devkit", "lib_deps") == [spec] + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_specs(lm.get_installed()) == [ + PackageSpec("DallasTemperature@3.8.1"), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + ] + # update package + result = clirunner.invoke( + package_update_cmd, + ["-e", "devkit", "-l", "milesburton/DallasTemperature@^3.8.0"], + ) + assert ProjectConfig().get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.8.0" + ] + # try again + result = clirunner.invoke( + package_update_cmd, + ["-e", "devkit", "-l", "milesburton/DallasTemperature@^3.8.0"], + ) + validate_cliresult(result) + assert "Already up-to-date." in result.output + + # install library without saving to config + result = clirunner.invoke( + package_update_cmd, + ["-e", "devkit", "-l", "milesburton/DallasTemperature@^3", "--no-save"], + ) + validate_cliresult(result) + assert "Already up-to-date." in result.output + config = ProjectConfig() + lm = LibraryPackageManager( + os.path.join(config.get("platformio", "libdeps_dir"), "devkit") + ) + assert pkgs_to_specs(lm.get_installed()) == [ + PackageSpec( + "DallasTemperature@%s" + % get_pkg_latest_version("milesburton/DallasTemperature") + ), + PackageSpec( + "OneWire@%s" % get_pkg_latest_version("paulstoffregen/OneWire") + ), + ] + assert config.get("env:devkit", "lib_deps") == [ + "milesburton/DallasTemperature@^3.8.0" + ] + + # unknown libraries + result = clirunner.invoke( + package_update_cmd, ["-l", "platformio/unknown_library"] + ) + assert isinstance(result.exception, UnknownPackageError) + + +def test_custom_project_tools( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) + spec = "toolchain-atmelavr@~1.50400.0" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-t", spec], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + assert ProjectConfig().get("env:devkit", "platform_packages") == [ + "platformio/toolchain-atmelavr@~1.50400.0" + ] + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("toolchain-atmelavr@1.50400.190710") + ] + result = clirunner.invoke( + package_update_cmd, + ["-e", "devkit", "-t", "toolchain-atmelavr@^1"], + ) + validate_cliresult(result) + assert ProjectConfig().get("env:devkit", "platform_packages") == [ + "platformio/toolchain-atmelavr@^1" + ] + assert pkgs_to_specs(ToolPackageManager().get_installed()) == [ + PackageSpec("toolchain-atmelavr@1.70300.191015") + ] + + # install without saving to config + result = clirunner.invoke( + package_update_cmd, + ["-e", "devkit", "-t", "toolchain-atmelavr@~1.70300.191015", "--no-save"], + ) + validate_cliresult(result) + assert "Already up-to-date." in result.output + assert ProjectConfig().get("env:devkit", "platform_packages") == [ + "platformio/toolchain-atmelavr@^1" + ] + + # unknown tool + result = clirunner.invoke(package_update_cmd, ["-t", "platformio/unknown_tool"]) + assert isinstance(result.exception, UnknownPackageError) + + +def test_custom_project_platforms( + clirunner, validate_cliresult, func_isolated_pio_core, tmp_path +): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_OUTDATED_CONFIG_TPL) + spec = "atmelavr@^2" + result = clirunner.invoke( + package_install_cmd, + ["-d", str(project_dir), "-e", "devkit", "-p", spec, "--skip-dependencies"], + ) + validate_cliresult(result) + with fs.cd(str(project_dir)): + assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ + PackageSpec("atmelavr@2.2.0") + ] + assert ProjectConfig().get("env:devkit", "platform") == "platformio/atmelavr@^2" + + # update + result = clirunner.invoke( + package_install_cmd, + ["-e", "devkit", "-p", "platformio/atmelavr@^3", "--skip-dependencies"], + ) + validate_cliresult(result) + assert pkgs_to_specs(PlatformPackageManager().get_installed()) == [ + PackageSpec("atmelavr@3.4.0"), + PackageSpec("atmelavr@2.2.0"), + ] + assert ProjectConfig().get("env:devkit", "platform") == "platformio/atmelavr@^2" + + # unknown platform + result = clirunner.invoke(package_install_cmd, ["-p", "unknown_platform"]) + assert isinstance(result.exception, UnknownPackageError) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py new file mode 100644 index 0000000..4f23ff9 --- /dev/null +++ b/tests/commands/test_account_org_team.py @@ -0,0 +1,524 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=global-statement,unused-argument + +import json +import os +import random + +import pytest +import requests + +from platformio.account.cli import cli as cmd_account +from platformio.account.org.cli import cli as cmd_org +from platformio.account.team.cli import cli as cmd_team + +pytestmark = pytest.mark.skipif( + not all( + os.environ.get(name) + for name in ( + "TEST_EMAIL_LOGIN", + "TEST_EMAIL_PASSWORD", + "TEST_EMAIL_IMAP_SERVER", + ) + ), + reason=( + "requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD, " + "and TEST_EMAIL_IMAP_SERVER environment variables" + ), +) + +USER_NAME = "test-piocore-%s" % str(random.randint(0, 100000)) +USER_EMAIL = os.environ.get("TEST_EMAIL_LOGIN", "").replace("@", f"+{USER_NAME}@") +USER_PASSWORD = f"Qwerty-{random.randint(0, 100000)}" +USER_FIRST_NAME = "FirstName" +USER_LAST_NAME = "LastName" + +ORG_NAME = "testorg-piocore-%s" % str(random.randint(0, 100000)) +ORG_DISPLAY_NAME = "Test Org for PIO Core" +EXISTING_OWNER = "piolabs" + +TEAM_NAME = "test-" + str(random.randint(0, 100000)) +TEAM_DESCRIPTION = "team for CI test" + + +def verify_account(email_contents): + link = ( + email_contents.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + with requests.Session() as session: + result = session.get(link).text + link = result.split(' 0 + assert result.exception + assert "You are not authorized!" in str(result.exception) + + result = clirunner.invoke( + cmd_account, + ["login", "-u", USER_NAME, "-p", USER_PASSWORD], + ) + validate_cliresult(result) + + +# def _test_account_destroy_with_linked_resources( +# clirunner, validate_cliresult, receive_email, isolated_pio_core, tmpdir_factory +# ): +# package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" +# +# tmp_dir = tmpdir_factory.mktemp("package") +# fd = FileDownloader(package_url, str(tmp_dir)) +# pkg_dir = tmp_dir.mkdir("raw_package") +# fd.start(with_progress=False, silent=True) +# with FileUnpacker(fd.get_filepath()) as unpacker: +# unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) +# +# result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) +# validate_cliresult(result) +# try: +# result = receive_email(email) +# assert "Congrats" in result +# assert "was published" in result +# except: # pylint:disable=bare-except +# pass +# +# result = clirunner.invoke(cmd_account, ["destroy"], "y") +# assert result.exit_code != 0 +# assert ( +# "We can not destroy the %s account due to 1 linked resources from registry" +# % username +# ) +# +# result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) +# validate_cliresult(result) + + +def test_org_create(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_org, + ["create", "--email", USER_EMAIL, "--displayname", ORG_DISPLAY_NAME, ORG_NAME], + ) + validate_cliresult(result) + + +def test_org_list(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result == [ + { + "orgname": ORG_NAME, + "displayname": ORG_DISPLAY_NAME, + "email": USER_EMAIL, + "owners": [ + { + "username": USER_NAME, + "firstname": USER_FIRST_NAME, + "lastname": USER_LAST_NAME, + } + ], + } + ] + + +def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke(cmd_org, ["add", ORG_NAME, EXISTING_OWNER]) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + assert EXISTING_OWNER in result.output + + +def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke(cmd_org, ["remove", ORG_NAME, EXISTING_OWNER]) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + assert EXISTING_OWNER not in result.output + + +def test_org_update(clirunner, validate_cliresult, isolated_pio_core): + new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) + new_display_name = "Test Org for PIO Core" + + result = clirunner.invoke( + cmd_org, + [ + "update", + ORG_NAME, + "--orgname", + new_orgname, + "--displayname", + new_display_name, + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result == [ + { + "orgname": new_orgname, + "displayname": new_display_name, + "email": USER_EMAIL, + "owners": [ + { + "username": USER_NAME, + "firstname": USER_FIRST_NAME, + "lastname": USER_LAST_NAME, + } + ], + } + ] + + result = clirunner.invoke( + cmd_org, + [ + "update", + new_orgname, + "--orgname", + ORG_NAME, + "--displayname", + ORG_DISPLAY_NAME, + ], + ) + validate_cliresult(result) + + +def test_team_create(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_team, + [ + "create", + "%s:%s" % (ORG_NAME, TEAM_NAME), + "--description", + TEAM_DESCRIPTION, + ], + ) + validate_cliresult(result) + + +def test_team_list(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_team, + ["list", "%s" % ORG_NAME, "--json-output"], + ) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + for item in json_result: + del item["id"] + assert json_result == [ + {"name": TEAM_NAME, "description": TEAM_DESCRIPTION, "members": []} + ] + + +def _test_team_add_member(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_team, + ["add", "%s:%s" % (ORG_NAME, TEAM_NAME), EXISTING_OWNER], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_team, + ["list", "%s" % ORG_NAME, "--json-output"], + ) + validate_cliresult(result) + assert EXISTING_OWNER in result.output + + +def _test_team_remove(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_team, + ["remove", "%s:%s" % (ORG_NAME, TEAM_NAME), EXISTING_OWNER], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_team, + ["list", "%s" % ORG_NAME, "--json-output"], + ) + validate_cliresult(result) + assert EXISTING_OWNER not in result.output + + +def _test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_core): + new_teamname = "new-" + str(random.randint(0, 100000)) + newteam_description = "Updated Description" + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (ORG_NAME, TEAM_NAME), + "--name", + new_teamname, + "--description", + newteam_description, + ], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_team, + ["list", "%s" % ORG_NAME, "--json-output"], + ) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + for item in json_result: + del item["id"] + assert json_result == [ + {"name": new_teamname, "description": newteam_description, "members": []} + ] + + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (ORG_NAME, new_teamname), + "--name", + TEAM_NAME, + "--description", + TEAM_DESCRIPTION, + ], + ) + validate_cliresult(result) + + +def test_cleanup(clirunner, validate_cliresult, receive_email, isolated_pio_core): + result = clirunner.invoke( + cmd_team, ["destroy", "%s:%s" % (ORG_NAME, TEAM_NAME)], "y" + ) + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["destroy", ORG_NAME], "y") + validate_cliresult(result) + result = clirunner.invoke(cmd_account, ["destroy"], "y") + validate_cliresult(result) diff --git a/tests/commands/test_boards.py b/tests/commands/test_boards.py new file mode 100644 index 0000000..244f0b7 --- /dev/null +++ b/tests/commands/test_boards.py @@ -0,0 +1,52 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from platformio.commands.boards import cli as cmd_boards +from platformio.commands.platform import platform_search as cmd_platform_search + + +def test_board_json_output(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_boards, ["mbed", "--json-output"]) + validate_cliresult(result) + boards = json.loads(result.output) + assert isinstance(boards, list) + assert any("mbed" in b["frameworks"] for b in boards) + + +def test_board_raw_output(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_boards, ["espidf"]) + validate_cliresult(result) + assert "espressif32" in result.output + + +def test_board_options(clirunner, validate_cliresult): + required_opts = set(["fcpu", "frameworks", "id", "mcu", "name", "platform"]) + + # fetch available platforms + result = clirunner.invoke(cmd_platform_search, ["--json-output"]) + validate_cliresult(result) + search_result = json.loads(result.output) + assert isinstance(search_result, list) + assert search_result + platforms = [item["name"] for item in search_result] + + result = clirunner.invoke(cmd_boards, ["mbed", "--json-output"]) + validate_cliresult(result) + boards = json.loads(result.output) + + for board in boards: + assert required_opts.issubset(set(board)) + assert board["platform"] in platforms diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py new file mode 100644 index 0000000..5403ee0 --- /dev/null +++ b/tests/commands/test_check.py @@ -0,0 +1,759 @@ +# Copyright (c) 2019-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-outer-name + +import json +import os +import sys + +import pytest + +from platformio import fs +from platformio.check.cli import cli as cmd_check + +DEFAULT_CONFIG = """ +[env:native] +platform = native +""" + +TEST_CODE = """ +#include + +void run_defects() { + /* Freeing a pointer twice */ + int* doubleFreePi = (int*)malloc(sizeof(int)); + *doubleFreePi=2; + free(doubleFreePi); + free(doubleFreePi); /* High */ + + /* Reading uninitialized memory */ + int* uninitializedPi = (int*)malloc(sizeof(int)); + *uninitializedPi++; /* High + Medium*/ + free(uninitializedPi); + + /* Delete instead of delete [] */ + int* wrongDeletePi = new int[10]; + wrongDeletePi++; + delete wrongDeletePi; /* High */ + + /* Index out of bounds */ + int arr[10]; + for(int i=0; i < 11; i++) { + arr[i] = 0; /* High */ + } +} + +int main() { + int uninitializedVar; /* Low */ + run_defects(); +} +""" + + +PVS_STUDIO_FREE_LICENSE_HEADER = """ +// This is an open source non-commercial project. Dear PVS-Studio, please check it. +// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com +""" + +EXPECTED_ERRORS = 5 +EXPECTED_WARNINGS = 1 +EXPECTED_STYLE = 4 +EXPECTED_DEFECTS = EXPECTED_ERRORS + EXPECTED_WARNINGS + EXPECTED_STYLE + + +@pytest.fixture(scope="module") +def check_dir(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + return tmpdir + + +def count_defects(output): + error, warning, style = 0, 0, 0 + for line in output.split("\n"): + if "[high:error]" in line: + error += 1 + elif "[medium:warning]" in line: + warning += 1 + elif "[low:style]" in line: + style += 1 + return error, warning, style + + +def test_check_cli_output(clirunner, validate_cliresult, check_dir): + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS + + +def test_check_json_output(clirunner, validate_cliresult, check_dir): + result = clirunner.invoke( + cmd_check, ["--project-dir", str(check_dir), "--json-output"] + ) + validate_cliresult(result) + + output = json.loads(result.stdout.strip()) + + assert isinstance(output, list) + assert len(output[0].get("defects", [])) == EXPECTED_DEFECTS + + +def test_check_tool_defines_passed(clirunner, check_dir): + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) + output = result.output + + assert "PLATFORMIO=" in output + assert "__GNUC__" in output + + +def test_check_tool_complex_defines_handled( + clirunner, validate_cliresult, tmpdir_factory +): + project_dir = tmpdir_factory.mktemp("project_dir") + + project_dir.join("platformio.ini").write( + DEFAULT_CONFIG + + R""" +check_tool = cppcheck, clangtidy, pvs-studio +build_flags = + -DEXTERNAL_INCLUDE_FILE=\"test.h\" + "-DDEFINE_WITH_SPACE="Hello World!"" +""" + ) + + src_dir = project_dir.mkdir("src") + src_dir.join("test.h").write( + """ +#ifndef TEST_H +#define TEST_H +#define ARBITRARY_CONST_VALUE 10 +#endif +""" + ) + + src_dir.join("main.c").write( + PVS_STUDIO_FREE_LICENSE_HEADER + + """ +#if !defined(EXTERNAL_INCLUDE_FILE) +#error "EXTERNAL_INCLUDE_FILE is not declared!" +#else +#include EXTERNAL_INCLUDE_FILE +#endif + +int main() +{ + /* Index out of bounds */ + int arr[ARBITRARY_CONST_VALUE]; + for(int i=0; i < ARBITRARY_CONST_VALUE+1; i++) { + arr[i] = 0; /* High */ + } + return 0; +} +""" + ) + + default_result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir)]) + validate_cliresult(default_result) + + +def test_check_language_standard_definition_passed(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\nbuild_flags = -std=c++17" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "__cplusplus=201703L" in result.output + assert "--std=c++17" in result.output + + +def test_check_language_standard_option_is_converted(clirunner, tmpdir): + config = ( + DEFAULT_CONFIG + + """ +build_flags = -std=gnu++1y + """ + ) + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "--std=c++14" in result.output + + +def test_check_language_standard_is_prioritized_over_build_flags(clirunner, tmpdir): + config = ( + DEFAULT_CONFIG + + """ +check_flags = --std=c++03 +build_flags = -std=c++17 + """ + ) + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "--std=c++03" in result.output + assert "--std=c++17" not in result.output + + +def test_check_language_standard_for_c_language(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\nbuild_flags = -std=c11" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "--std=c11" in result.output + assert "__STDC_VERSION__=201112L" in result.output + assert "__cplusplus" not in result.output + + +def test_check_severity_threshold(clirunner, validate_cliresult, check_dir): + result = clirunner.invoke( + cmd_check, ["--project-dir", str(check_dir), "--severity=high"] + ) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors == EXPECTED_ERRORS + assert warnings == 0 + assert style == 0 + + +def test_check_includes_passed(clirunner, check_dir): + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) + + inc_count = 0 + for line in result.output.split("\n"): + if line.startswith("Includes:"): + inc_count = line.count("-I") + + # at least 1 include path for default mode + assert inc_count > 0 + + +def test_check_silent_mode(clirunner, validate_cliresult, check_dir): + result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--silent"]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors == EXPECTED_ERRORS + assert warnings == 0 + assert style == 0 + + +def test_check_no_source_files(clirunner, tmpdir): + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + tmpdir.mkdir("src") + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code != 0 + assert errors == 0 + assert warnings == 0 + assert style == 0 + + +def test_check_bad_flag_passed(clirunner, check_dir): + result = clirunner.invoke( + cmd_check, ["--project-dir", str(check_dir), '"--flags=--UNKNOWN"'] + ) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code != 0 + assert errors == 0 + assert warnings == 0 + assert style == 0 + + +def test_check_success_if_no_errors(clirunner, validate_cliresult, tmpdir): + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + tmpdir.mkdir("src").join("main.c").write( + """ +#include + +void unused_function(){ + int unusedVar = 0; + int* iP = &unusedVar; + *iP++; +} + +int main() { +} +""" + ) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert "[PASSED]" in result.output + assert errors == 0 + assert warnings == 1 + assert style == 1 + + +def test_check_individual_flags_passed(clirunner, validate_cliresult, tmpdir): + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy, pvs-studio" + config += """\ncheck_flags = + cppcheck: --std=c++11 + clangtidy: --fix-errors + pvs-studio: --analysis-mode=4 +""" + + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write( + PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE + ) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + validate_cliresult(result) + + clang_flags_found = cppcheck_flags_found = pvs_flags_found = False + for l in result.output.split("\n"): + if "--fix" in l and "clang-tidy" in l and "--std=c++11" not in l: + clang_flags_found = True + elif "--std=c++11" in l and "cppcheck" in l and "--fix" not in l: + cppcheck_flags_found = True + elif ( + "--analysis-mode=4" in l and "pvs-studio" in l.lower() and "--fix" not in l + ): + pvs_flags_found = True + + assert clang_flags_found + assert cppcheck_flags_found + assert pvs_flags_found + + +def test_check_cppcheck_misra_addon(clirunner, validate_cliresult, tmpdir_factory): + check_dir = tmpdir_factory.mktemp("project") + check_dir.join("platformio.ini").write(DEFAULT_CONFIG) + check_dir.mkdir("src").join("main.c").write(TEST_CODE) + check_dir.join("misra.json").write( + """ +{ + "script": "addons/misra.py", + "args": ["--rule-texts=rules.txt"] +} +""" + ) + + check_dir.join("rules.txt").write( + """ +Appendix A Summary of guidelines +Rule 3.1 Required +R3.1 text. +Rule 4.1 Required +R4.1 text. +Rule 10.4 Mandatory +R10.4 text. +Rule 11.5 Advisory +R11.5 text. +Rule 15.5 Advisory +R15.5 text. +Rule 15.6 Required +R15.6 text. +Rule 17.7 Required +R17.7 text. +Rule 20.1 Advisory +R20.1 text. +Rule 21.3 Required +R21.3 Found MISRA defect +Rule 21.4 +R21.4 text. +""" + ) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(check_dir), "--flags=--addon=misra.json"] + ) + + validate_cliresult(result) + assert "R21.3 Found MISRA defect" in result.output + assert not os.path.isfile(os.path.join(str(check_dir), "src", "main.cpp.dump")) + + +def test_check_fails_on_defects_only_with_flag(clirunner, validate_cliresult, tmpdir): + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + + default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + + result_with_flag = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] + ) + + validate_cliresult(default_result) + assert result_with_flag.exit_code != 0 + + +def test_check_fails_on_defects_only_on_specified_level( + clirunner, validate_cliresult, tmpdir +): + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write( + """ +#include + +void unused_function(){ + int unusedVar = 0; + int* iP = &unusedVar; + *iP++; +} + +int main() { +} +""" + ) + + high_result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] + ) + validate_cliresult(high_result) + + low_result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=low"] + ) + + assert low_result.exit_code != 0 + + +def test_check_pvs_studio_free_license(clirunner, tmpdir): + config = """ +[env:test] +platform = teensy +board = teensy35 +framework = arduino +check_tool = pvs-studio +""" + + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high", "-v"] + ) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code != 0 + assert errors != 0 + assert warnings != 0 + assert style == 0 + + +def test_check_pvs_studio_fails_without_license(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\ncheck_tool = pvs-studio" + + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write(TEST_CODE) + + default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + verbose_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert default_result.exit_code != 0 + assert "failed to perform check" in default_result.output.lower() + + assert verbose_result.exit_code != 0 + assert "license was not entered" in verbose_result.output.lower() + + +@pytest.mark.skipif( + sys.platform != "win32", + reason="For some reason the error message is different on Windows", +) +def test_check_pvs_studio_fails_broken_license(clirunner, tmpdir): + config = ( + DEFAULT_CONFIG + + """ +check_tool = pvs-studio +check_flags = --lic-file=./pvs-studio.lic +""" + ) + + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write(TEST_CODE) + tmpdir.join("pvs-studio.lic").write( + """ +TEST +TEST-TEST-TEST-TEST +""" + ) + + default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + verbose_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert default_result.exit_code != 0 + assert "failed to perform check" in default_result.output.lower() + + assert verbose_result.exit_code != 0 + assert "license information is incorrect" in verbose_result.output.lower() + + +@pytest.mark.parametrize("framework", ["arduino", "stm32cube", "zephyr"]) +@pytest.mark.parametrize("check_tool", ["cppcheck", "clangtidy", "pvs-studio"]) +def test_check_embedded_platform_all_tools( + clirunner, validate_cliresult, tmpdir, framework, check_tool +): + config = f""" +[env:test] +platform = ststm32 +board = nucleo_f401re +framework = {framework} +check_tool = {check_tool} +""" + tmpdir.mkdir("src").join("main.c").write( + PVS_STUDIO_FREE_LICENSE_HEADER + + """ +#include + +void unused_function(int val){ + int unusedVar = 0; + int* iP = &unusedVar; + *iP++; +} + +int main() { +} +""" + ) + + tmpdir.join("platformio.ini").write(config) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + defects = sum(count_defects(result.output)) + assert defects > 0, "Not defects were found!" + + +def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir): + config = """ +[env:test] +platform = nordicnrf52 +board = nrf52_dk +framework = arduino +""" + + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--skip-packages", "-v"] + ) + validate_cliresult(result) + + project_path = fs.to_unix_path(str(tmpdir)) + for line in result.output.split("\n"): + if not line.startswith("Includes:"): + continue + for inc in line.split(" "): + if inc.startswith("-I") and project_path not in inc: + pytest.fail("Detected an include path from packages: " + inc) + + +def test_check_multiline_error(clirunner, tmpdir_factory): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + project_dir.mkdir("include").join("main.h").write( + """ +#error This is a multiline error message \\ +that should be correctly reported \\ +in both default and verbose modes. +""" + ) + + project_dir.mkdir("src").join("main.c").write( + """ +#include +#include "main.h" + +int main() {} +""" + ) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir)]) + errors, _, _ = count_defects(result.output) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir), "-v"]) + verbose_errors, _, _ = count_defects(result.output) + + assert verbose_errors == errors == 1 + + +def test_check_handles_spaces_in_paths(clirunner, validate_cliresult, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project dir") + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy, pvs-studio" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write( + PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE + ) + + default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + + validate_cliresult(default_result) + + +# +# Files filtering functionality +# + + +@pytest.mark.parametrize( + "src_filter,number_of_checked_files", + [ + (["+"], 1), + (["+"], 1), + (["+", "-"], 2), + (["-<*> + + +"], 3), + ], + ids=["Single file", "Glob pattern", "Exclude pattern", "Filter as string"], +) +def test_check_src_filter( + clirunner, + validate_cliresult, + tmpdir_factory, + src_filter, + number_of_checked_files, +): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.join("app.cpp").write(TEST_CODE) + src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + cmd_args = ["--project-dir", str(tmpdir)] + [ + "--src-filters=%s" % f for f in src_filter + ] + + result = clirunner.invoke(cmd_check, cmd_args) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * number_of_checked_files + + +def test_check_src_filter_from_config(clirunner, validate_cliresult, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + + config = ( + DEFAULT_CONFIG + + """ +check_src_filters = + + + + + """ + ) + tmpdir.join("platformio.ini").write(config) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + assert "main.cpp" not in result.output + + +def test_check_custom_pattern_absolute_path_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + check_dir = tmpdir_factory.mktemp("custom_src_dir") + check_dir.join("main.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] + ) + + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors == EXPECTED_ERRORS + assert warnings == EXPECTED_WARNINGS + assert style == EXPECTED_STYLE + + +def test_check_custom_pattern_relative_path_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, + ["--project-dir", str(tmpdir), "--pattern=src/uart", "--pattern=src/spi"], + ) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + + +def test_check_src_filter_from_config_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + tmpdir = tmpdir_factory.mktemp("project") + + config = ( + DEFAULT_CONFIG + + """ +check_patterns = + src/spi/*.c* + tests/test.cpp + """ + ) + tmpdir.join("platformio.ini").write(config) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + assert "main.cpp" not in result.output diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py new file mode 100644 index 0000000..f67bada --- /dev/null +++ b/tests/commands/test_ci.py @@ -0,0 +1,193 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os.path import isfile, join + +from platformio.commands.ci import cli as cmd_ci +from platformio.package.commands.install import package_install_cmd + + +def test_ci_empty(clirunner): + result = clirunner.invoke(cmd_ci) + assert result.exit_code != 0 + assert "Invalid value: Missing argument 'src'" in result.output + + +def test_ci_boards(clirunner, validate_cliresult): + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src", "main.cpp"), + "-b", + "uno", + "-b", + "leonardo", + ], + ) + validate_cliresult(result) + + +def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src", "main.cpp"), + "-b", + "uno", + "--build-dir", + build_dir, + ], + ) + validate_cliresult(result) + assert not isfile(join(build_dir, "platformio.ini")) + + +def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src", "main.cpp"), + "-b", + "uno", + "--build-dir", + build_dir, + "--keep-build-dir", + ], + ) + validate_cliresult(result) + assert isfile(join(build_dir, "platformio.ini")) + + # 2nd attempt + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src", "main.cpp"), + "-b", + "metro", + "--build-dir", + build_dir, + "--keep-build-dir", + ], + ) + validate_cliresult(result) + + assert "board: uno" in result.output + assert "board: metro" in result.output + + +def test_ci_keep_build_dir_single_src_dir( + clirunner, tmpdir_factory, validate_cliresult +): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + + # Run two times to detect possible "AlreadyExists" errors + for _ in range(2): + result = clirunner.invoke( + cmd_ci, + [ + join("examples", "wiring-blink", "src"), + "-b", + "uno", + "--build-dir", + build_dir, + "--keep-build-dir", + ], + ) + validate_cliresult(result) + + +def test_ci_keep_build_dir_nested_src_dirs( + clirunner, tmpdir_factory, validate_cliresult +): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + + # Split default Arduino project in two parts + src_dir1 = tmpdir_factory.mktemp("src_1") + src_dir1.join("src1.cpp").write( + """ +#include +void setup() {} +""" + ) + + src_dir2 = tmpdir_factory.mktemp("src_2") + src_dir2.join("src2.cpp").write( + """ +#include +void loop() {} +""" + ) + + src_dir1 = str(src_dir1) + src_dir2 = str(src_dir2) + + # Run two times to detect possible "AlreadyExists" errors + for _ in range(2): + result = clirunner.invoke( + cmd_ci, + [ + src_dir1, + src_dir2, + "-b", + "teensy40", + "--build-dir", + build_dir, + "--keep-build-dir", + ], + ) + + validate_cliresult(result) + + +def test_ci_project_conf(clirunner, validate_cliresult): + project_dir = join("examples", "wiring-blink") + result = clirunner.invoke( + cmd_ci, + [ + join(project_dir, "src", "main.cpp"), + "--project-conf", + join(project_dir, "platformio.ini"), + ], + ) + validate_cliresult(result) + assert "uno" in result.output + + +def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): + storage_dir = str(tmpdir_factory.mktemp("lib")) + result = clirunner.invoke( + package_install_cmd, + ["--global", "--storage-dir", storage_dir, "--library", "1"], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_ci, + [ + join( + storage_dir, + "OneWire", + "examples", + "DS2408_Switch", + "DS2408_Switch.ino", + ), + "-l", + join(storage_dir, "OneWire"), + "-b", + "uno", + ], + ) + validate_cliresult(result) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py new file mode 100644 index 0000000..651cf57 --- /dev/null +++ b/tests/commands/test_init.py @@ -0,0 +1,217 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from platformio.commands.boards import cli as cmd_boards +from platformio.project.commands.init import project_init_cmd +from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectEnvsNotAvailableError + + +def validate_pioproject(pioproject_dir): + pioconf_path = os.path.join(pioproject_dir, "platformio.ini") + assert os.path.isfile(pioconf_path) and os.path.getsize(pioconf_path) > 0 + assert os.path.isdir(os.path.join(pioproject_dir, "src")) and os.path.isdir( + os.path.join(pioproject_dir, "lib") + ) + + +def test_init_default(clirunner, validate_cliresult): + with clirunner.isolated_filesystem(): + result = clirunner.invoke(project_init_cmd) + validate_cliresult(result) + validate_pioproject(os.getcwd()) + + +def test_init_ext_folder(clirunner, validate_cliresult): + with clirunner.isolated_filesystem(): + ext_folder_name = "ext_folder" + os.makedirs(ext_folder_name) + result = clirunner.invoke(project_init_cmd, ["-d", ext_folder_name]) + validate_cliresult(result) + validate_pioproject(os.path.join(os.getcwd(), ext_folder_name)) + + +def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir): + with tmpdir.as_cwd(): + for _ in range(2): + result = clirunner.invoke( + project_init_cmd, + ["-b", "uno", "-b", "uno", "--no-install-dependencies"], + ) + validate_cliresult(result) + validate_pioproject(str(tmpdir)) + config = ProjectConfig(os.path.join(os.getcwd(), "platformio.ini")) + config.validate() + assert set(config.sections()) == set(["env:uno"]) + + +def test_init_ide_without_board(clirunner, tmpdir): + with tmpdir.as_cwd(): + result = clirunner.invoke(project_init_cmd, ["--ide", "vscode"]) + assert result.exit_code != 0 + assert isinstance(result.exception, ProjectEnvsNotAvailableError) + + +def test_init_ide_vscode(clirunner, validate_cliresult, tmpdir): + with tmpdir.as_cwd(): + result = clirunner.invoke( + project_init_cmd, + [ + "--ide", + "vscode", + "-b", + "uno", + "-b", + "teensy31", + "--no-install-dependencies", + ], + ) + validate_cliresult(result) + validate_pioproject(str(tmpdir)) + assert all( + tmpdir.join(".vscode").join(f).check() + for f in ("c_cpp_properties.json", "launch.json") + ) + assert ( + "framework-arduino-avr" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) + + # switch to NodeMCU + result = clirunner.invoke( + project_init_cmd, + ["--ide", "vscode", "-b", "nodemcuv2", "--no-install-dependencies"], + ) + validate_cliresult(result) + validate_pioproject(str(tmpdir)) + assert ( + "framework-arduinoespressif8266" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) + + # switch to teensy31 via env name + result = clirunner.invoke( + project_init_cmd, + ["--ide", "vscode", "-e", "teensy31", "--no-install-dependencies"], + ) + validate_cliresult(result) + validate_pioproject(str(tmpdir)) + assert ( + "framework-arduinoteensy" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) + + # switch to the first board + result = clirunner.invoke( + project_init_cmd, ["--ide", "vscode", "--no-install-dependencies"] + ) + validate_cliresult(result) + validate_pioproject(str(tmpdir)) + assert ( + "framework-arduino-avr" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) + + +def test_init_ide_eclipse(clirunner, validate_cliresult): + with clirunner.isolated_filesystem(): + result = clirunner.invoke( + project_init_cmd, + ["-b", "uno", "--ide", "eclipse", "--no-install-dependencies"], + ) + validate_cliresult(result) + validate_pioproject(os.getcwd()) + assert all(os.path.isfile(f) for f in (".cproject", ".project")) + + +def test_init_special_board(clirunner, validate_cliresult): + with clirunner.isolated_filesystem(): + result = clirunner.invoke(project_init_cmd, ["-b", "uno"]) + validate_cliresult(result) + validate_pioproject(os.getcwd()) + + result = clirunner.invoke(cmd_boards, ["Arduino Uno", "--json-output"]) + validate_cliresult(result) + boards = json.loads(result.output) + + config = ProjectConfig(os.path.join(os.getcwd(), "platformio.ini")) + config.validate() + + expected_result = dict( + platform=str(boards[0]["platform"]), + board="uno", + framework=[str(boards[0]["frameworks"][0])], + ) + assert config.has_section("env:uno") + assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( + expected_result.items() + ) + + +def test_init_enable_auto_uploading(clirunner, validate_cliresult): + with clirunner.isolated_filesystem(): + result = clirunner.invoke( + project_init_cmd, + [ + "-b", + "uno", + "--project-option", + "targets=upload", + "--no-install-dependencies", + ], + ) + validate_cliresult(result) + validate_pioproject(os.getcwd()) + config = ProjectConfig(os.path.join(os.getcwd(), "platformio.ini")) + config.validate() + expected_result = dict( + targets=["upload"], platform="atmelavr", board="uno", framework=["arduino"] + ) + assert config.has_section("env:uno") + assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( + expected_result.items() + ) + + +def test_init_custom_framework(clirunner, validate_cliresult): + with clirunner.isolated_filesystem(): + result = clirunner.invoke( + project_init_cmd, + [ + "-b", + "teensy31", + "--project-option", + "framework=mbed", + "--no-install-dependencies", + ], + ) + validate_cliresult(result) + validate_pioproject(os.getcwd()) + config = ProjectConfig(os.path.join(os.getcwd(), "platformio.ini")) + config.validate() + expected_result = dict(platform="teensy", board="teensy31", framework=["mbed"]) + assert config.has_section("env:teensy31") + assert sorted(config.items(env="teensy31", as_dict=True).items()) == sorted( + expected_result.items() + ) + + +def test_init_incorrect_board(clirunner): + result = clirunner.invoke(project_init_cmd, ["-b", "missed_board"]) + assert result.exit_code == 2 + assert "Error: Invalid value for" in result.output + assert isinstance(result.exception, SystemExit) diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py new file mode 100644 index 0000000..8f47ad6 --- /dev/null +++ b/tests/commands/test_lib.py @@ -0,0 +1,234 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json +import os + +import pytest +import semantic_version + +from platformio.commands.lib import cli as cmd_lib +from platformio.package.meta import PackageType +from platformio.package.vcsclient import VCSClientFactory +from platformio.project.config import ProjectConfig +from platformio.registry.client import RegistryClient + + +def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): + regclient = RegistryClient() + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write( + """ +[env] +lib_deps = ArduinoJson + +[env:one] +board = devkit + +[env:two] +framework = foo +lib_deps = + CustomLib + ArduinoJson @ 5.10.1 +""" + ) + result = clirunner.invoke( + cmd_lib, + ["-d", str(project_dir), "install", "64", "knolleary/PubSubClient@~2.7"], + ) + validate_cliresult(result) + aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert sorted(config.get("env:one", "lib_deps")) == sorted( + [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + ] + ) + assert sorted(config.get("env:two", "lib_deps")) == sorted( + [ + "CustomLib", + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + ] + ) + + # ensure "build" version without NPM spec + result = clirunner.invoke( + cmd_lib, + ["-d", str(project_dir), "-e", "one", "install", "mbed-sam-grove/LinkedList"], + ) + validate_cliresult(result) + ll_pkg_data = regclient.get_package( + PackageType.LIBRARY, "mbed-sam-grove", "LinkedList" + ) + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert sorted(config.get("env:one", "lib_deps")) == sorted( + [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + ] + ) + + # check external package via Git repo + result = clirunner.invoke( + cmd_lib, + [ + "-d", + str(project_dir), + "-e", + "one", + "install", + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", + ], + ) + validate_cliresult(result) + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert len(config.get("env:one", "lib_deps")) == 4 + assert config.get("env:one", "lib_deps")[3] == ( + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3" + ) + + # test uninstalling + # from all envs + result = clirunner.invoke( + cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"] + ) + validate_cliresult(result) + # from "one" env + result = clirunner.invoke( + cmd_lib, + [ + "-d", + str(project_dir), + "-e", + "one", + "uninstall", + "knolleary/PubSubClient@~2.7", + ], + ) + validate_cliresult(result) + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert len(config.get("env:one", "lib_deps")) == 2 + assert len(config.get("env:two", "lib_deps")) == 2 + assert config.get("env:one", "lib_deps") == [ + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", + ] + assert config.get("env:two", "lib_deps") == [ + "CustomLib", + "knolleary/PubSubClient@~2.7", + ] + + # test list + result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) + validate_cliresult(result) + assert "AsyncMqttClient-esphome @ 0.8.3+sha.f5aa899" in result.stdout + result = clirunner.invoke( + cmd_lib, ["-d", str(project_dir), "list", "--json-output"] + ) + validate_cliresult(result) + data = {} + for key, value in json.loads(result.stdout).items(): + data[os.path.basename(key)] = value + ame_lib = next( + item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome" + ) + ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"]) + assert len(data["two"]) == 1 + assert data["two"][0]["name"] == "PubSubClient" + assert "__pkg_dir" in data["one"][0] + assert ( + ame_lib["__src_url"] + == "git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" + ) + assert ame_lib["version"] == ("0.8.3+sha.%s" % ame_vcs.get_current_revision()) + + +def test_update(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("test-updates") + result = clirunner.invoke( + cmd_lib, + ["-d", str(storage_dir), "install", "ArduinoJson @ 5.10.1", "Blynk @ ~0.5.0"], + ) + validate_cliresult(result) + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "--json-output"] + ) + validate_cliresult(result) + outdated = json.loads(result.stdout) + assert len(outdated) == 2 + # ArduinoJson + assert outdated[0]["version"] == "5.10.1" + assert outdated[0]["versionWanted"] is None + assert semantic_version.Version( + outdated[0]["versionLatest"] + ) > semantic_version.Version("6.16.0") + # Blynk + assert outdated[1]["version"] == "0.5.4" + assert outdated[1]["versionWanted"] is None + assert semantic_version.Version( + outdated[1]["versionLatest"] + ) > semantic_version.Version("0.6.0") + + # check with spec + result = clirunner.invoke( + cmd_lib, + [ + "-d", + str(storage_dir), + "update", + "--dry-run", + "--json-output", + "ArduinoJson @ ^5", + ], + ) + validate_cliresult(result) + outdated = json.loads(result.stdout) + assert outdated[0]["version"] == "5.10.1" + assert outdated[0]["versionWanted"] == "5.13.4" + assert semantic_version.Version( + outdated[0]["versionLatest"] + ) > semantic_version.Version("6.16.0") + # update with spec + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "--silent", "ArduinoJson @ ^5.10.1"] + ) + validate_cliresult(result) + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "list", "--json-output"] + ) + validate_cliresult(result) + items = json.loads(result.stdout) + assert len(items) == 2 + assert items[0]["version"] == "5.13.4" + assert items[1]["version"] == "0.5.4" + + # Check incompatible + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "ArduinoJson @ ^5"] + ) + with pytest.raises( + AssertionError, + match="This command is deprecated", + ): + validate_cliresult(result) + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "ArduinoJson @ ^5"] + ) + validate_cliresult(result) + assert "ArduinoJson@5.13.4 is already up-to-date" in result.stdout diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py new file mode 100644 index 0000000..eb56c5b --- /dev/null +++ b/tests/commands/test_lib_complex.py @@ -0,0 +1,344 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long + +import json +import re + +from platformio.cli import PlatformioCLI +from platformio.commands.lib import cli as cmd_lib +from platformio.package.exception import UnknownPackageError +from platformio.util import strip_ansi_codes + +PlatformioCLI.leftover_args = ["--json-output"] # hook for click + + +def test_search(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) + validate_cliresult(result) + match = re.search(r"Found\s+(\d+)\spackages", result.output) + assert int(match.group(1)) > 2 + + result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) + validate_cliresult(result) + match = re.search(r"Found\s+(\d+)\spackages", result.output) + assert int(match.group(1)) > 1 + + +def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "64", + "ArduinoJson@~5.10.0", + "547@2.2.4", + "AsyncMqttClient@<=0.8.2", + "Adafruit PN532@1.2.0", + ], + ) + validate_cliresult(result) + + # install unknown library + result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@5.10.1", + "NeoPixelBus", + "AsyncMqttClient", + "ESPAsyncTCP", + "AsyncTCP", + "Adafruit PN532", + "Adafruit BusIO", + ] + assert set(items1) == set(items2) + + +def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", + "SomeLib=https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) + validate_cliresult(result) + + # incorrect requirements + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", + ], + ) + assert result.exit_code != 0 + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "SomeLib", + "OneWire", + "ESP32WebServer", + ] + assert set(items1) >= set(items2) + + +def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/gioblu/PJON.git#3.0", + "https://github.com/gioblu/PJON.git#6.2", + "https://github.com/bblanchon/ArduinoJson.git", + "https://github.com/platformio/platformio-libmirror.git", + # "https://developer.mbed.org/users/simon/code/TextLCD/", + "https://github.com/knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", + ], + ) + validate_cliresult(result) + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "PJON", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "platformio-libmirror", + "PubSubClient", + ] + assert set(items1) >= set(items2) + + +def test_install_duplicates( # pylint: disable=unused-argument + clirunner, validate_cliresult, without_internet +): + # registry + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz", + ], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + # archive + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + # repository + result = clirunner.invoke( + cmd_lib, + ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + +def test_global_lib_list(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["-g", "list"]) + validate_cliresult(result) + assert all( + n in result.output + for n in ( + "required: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "ArduinoJson @ 5.10.1", + "required: git+https://github.com/gioblu/PJON.git#3.0", + "PJON @ 3.0.0+sha.1fb26f", + ) + ) + + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + assert all( + n in result.output + for n in ( + "__pkg_dir", + '"__src_url": "git+https://github.com/gioblu/PJON.git#6.2"', + '"version": "5.10.1"', + ) + ) + items1 = [i["name"] for i in json.loads(result.output)] + items2 = [ + "Adafruit BusIO", + "Adafruit PN532", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "AsyncMqttClient", + "AsyncTCP", + "DallasTemperature", + "ESP32WebServer", + "ESPAsyncTCP", + "NeoPixelBus", + "OneWire", + "PJON", + "PJON", + "platformio-libmirror", + "PubSubClient", + ] + assert sorted(items1) == sorted(items2) + + versions1 = [ + "{name}@{version}".format(**item) for item in json.loads(result.output) + ] + versions2 = [ + "ArduinoJson@5.8.2", + "ArduinoJson@5.10.1", + "AsyncMqttClient@0.8.2", + "NeoPixelBus@2.2.4", + "PJON@6.2.0+sha.07fe9aa", + "PJON@3.0.0+sha.1fb26fd", + "PubSubClient@2.6.0+sha.bef5814", + "Adafruit PN532@1.2.0", + ] + assert set(versions1) >= set(versions2) + + +def test_global_lib_update_check(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"]) + validate_cliresult(result) + output = json.loads(result.output) + assert set( + ["Adafruit PN532", "AsyncMqttClient", "ESPAsyncTCP", "NeoPixelBus"] + ) == set(lib["name"] for lib in output) + + +def test_global_lib_update(clirunner, validate_cliresult): + # update library using package directory + result = clirunner.invoke( + cmd_lib, ["-g", "update", "NeoPixelBus", "--dry-run", "--json-output"] + ) + validate_cliresult(result) + oudated = json.loads(result.output) + assert len(oudated) == 1 + assert "__pkg_dir" in oudated[0] + result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) + validate_cliresult(result) + assert "Removing NeoPixelBus @ 2.2.4" in strip_ansi_codes(result.output) + + # update all libraries + result = clirunner.invoke( + cmd_lib, + ["-g", "update", "adafruit/Adafruit PN532", "marvinroger/AsyncMqttClient"], + ) + validate_cliresult(result) + + # update unknown library + result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): + # uninstall using package directory + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + validate_cliresult(result) + items = json.loads(result.output) + items = sorted(items, key=lambda item: item["__pkg_dir"]) + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) + validate_cliresult(result) + assert "Removing %s" % items[0]["name"] in strip_ansi_codes(result.output) + + # uninstall the rest libraries + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "uninstall", + "OneWire", + "https://github.com/bblanchon/ArduinoJson.git", + "ArduinoJson@!=5.6.7", + "Adafruit PN532", + ], + ) + validate_cliresult(result) + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "AsyncMqttClient", + "platformio-libmirror", + "PubSubClient", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "ESPAsyncTCP@1.2.0", + "AsyncTCP", + "ArduinoJson", + "ESPAsyncTCP", + "ESP32WebServer", + "PJON", + "NeoPixelBus", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "SomeLib", + ] + assert set(items1) == set(items2) + + # uninstall unknown library + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_lib_show(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["show", "64"]) + validate_cliresult(result) + assert all(s in result.output for s in ("ArduinoJson", "Arduino")) + result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) + validate_cliresult(result) + assert "OneWire" in result.output + + +def test_lib_builtin(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["builtin"]) + validate_cliresult(result) + result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) + validate_cliresult(result) + + +def test_lib_stats(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) + validate_cliresult(result) + assert set( + [ + "dlweek", + "added", + "updated", + "topkeywords", + "dlmonth", + "dlday", + "lastkeywords", + ] + ) == set(json.loads(result.output).keys()) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py new file mode 100644 index 0000000..967bf51 --- /dev/null +++ b/tests/commands/test_platform.py @@ -0,0 +1,141 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json +import os + +from platformio.commands import platform as cli_platform +from platformio.package.exception import UnknownPackageError +from platformio.util import strip_ansi_codes + + +def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.platform_search, ["arduino", "--json-output"] + ) + validate_cliresult(result) + search_result = json.loads(result.output) + assert isinstance(search_result, list) + assert search_result + platforms = [item["name"] for item in search_result] + assert "atmelsam" in platforms + + +def test_search_raw_output(clirunner, validate_cliresult): + result = clirunner.invoke(cli_platform.platform_search, ["arduino"]) + validate_cliresult(result) + assert "atmelavr" in result.output + + +def test_install_unknown_version(clirunner): + result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_install_unknown_from_registry(clirunner): + result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +# def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): +# result = clirunner.invoke( +# cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], +# ) +# assert result.exit_code != 0 +# assert isinstance(result.exception, IncompatiblePlatform) + + +def test_install_core_3_dev_platform(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.platform_install, + ["atmelavr@1.2.0", "--skip-default-package"], + ) + assert result.exit_code == 0 + + +def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.platform_install, + ["atmelavr@2.0.0", "--skip-default-package", "--with-package", "tool-avrdude"], + ) + validate_cliresult(result) + output = strip_ansi_codes(result.output) + assert "atmelavr @ 2.0.0" in output + assert not os.path.isdir(str(isolated_pio_core.join("packages"))) + + +def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.platform_install, + [ + "https://github.com/platformio/platform-espressif8266.git", + "--skip-default-package", + ], + ) + validate_cliresult(result) + assert "espressif8266" in result.output + assert not os.path.isdir(str(isolated_pio_core.join("packages"))) + + +def test_list_json_output(clirunner, validate_cliresult): + result = clirunner.invoke(cli_platform.platform_list, ["--json-output"]) + validate_cliresult(result) + list_result = json.loads(result.output) + assert isinstance(list_result, list) + assert list_result + platforms = [item["name"] for item in list_result] + assert set(["atmelavr", "espressif8266"]) == set(platforms) + + +def test_list_raw_output(clirunner, validate_cliresult): + result = clirunner.invoke(cli_platform.platform_list) + validate_cliresult(result) + assert all(s in result.output for s in ("atmelavr", "espressif8266")) + + +def test_update_check(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.package_install_cmd, + ["--global", "--tool", "platformio/tool-avrdude@~1.60300.0"], + ) + validate_cliresult(result) + result = clirunner.invoke( + cli_platform.platform_update, ["--dry-run", "--json-output"] + ) + validate_cliresult(result) + output = json.loads(result.output) + assert len(output) == 1 + assert output[0]["name"] == "atmelavr" + assert len(isolated_pio_core.join("packages").listdir()) == 1 + + +def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke(cli_platform.platform_update, ["atmelavr"]) + validate_cliresult(result) + output = strip_ansi_codes(result.output) + assert "Removing atmelavr @ 2.0.0" in output + assert "Platform Manager: Installing platformio/atmelavr @" in output + assert len(isolated_pio_core.join("packages").listdir()) == 2 + + +def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.platform_uninstall, ["atmelavr@1.2.0", "atmelavr", "espressif8266"] + ) + validate_cliresult(result) + assert not isolated_pio_core.join("platforms").listdir() diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py new file mode 100644 index 0000000..820de44 --- /dev/null +++ b/tests/commands/test_run.py @@ -0,0 +1,446 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from platformio.run.cli import cli as cmd_run + + +def test_generic_build(clirunner, validate_cliresult, tmpdir): + build_flags = [ + ("-D TEST_INT=13", "-DTEST_INT=13"), + ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), + ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'), + ("-Iinclude", "-Iinclude"), + ("-include cpppath-include.h", "cpppath-include.h"), + ("-Iextra_inc", "-Iextra_inc"), + ("-Inon-existing-dir", "non-existing-dir"), + ( + "-include $PROJECT_DIR/lib/component/component-forced-include.h", + "component-forced-include.h", + ), + ] + + tmpdir.join("platformio.ini").write( + """ +[env:native] +platform = native +extra_scripts = + pre:pre_script.py + post_script.py +lib_ldf_mode = deep+ +build_src_flags = -DI_AM_ONLY_SRC_FLAG +build_flags = + ; -DCOMMENTED_MACRO + %s ; inline comment + """ + % " ".join([f[0] for f in build_flags]) + ) + + tmpdir.join("pre_script.py").write( + """ +Import("env") + +def post_prog_action(source, target, env): + print("post_prog_action is called") + +env.AddPostAction("$PROGPATH", post_prog_action) + """ + ) + tmpdir.join("post_script.py").write( + """ +Import("projenv") + +projenv.Append(CPPDEFINES="POST_SCRIPT_MACRO") + """ + ) + + tmpdir.mkdir("extra_inc").join("foo.h").write( + """ +#define FOO + """ + ) + + tmpdir.mkdir("src").join("main.cpp").write( + """ +#include "foo.h" + +#ifndef FOO +#error "FOO" +#endif + +#ifdef I_AM_ONLY_SRC_FLAG +#include +#else +#error "I_AM_ONLY_SRC_FLAG" +#endif + +#if !defined(TEST_INT) || TEST_INT != 13 +#error "TEST_INT" +#endif + +#ifndef TEST_STR_SPACE +#error "TEST_STR_SPACE" +#endif + +#ifndef I_AM_COMPONENT +#error "I_AM_COMPONENT" +#endif + +#ifndef POST_SCRIPT_MACRO +#error "POST_SCRIPT_MACRO" +#endif + +#ifndef I_AM_FORCED_COMPONENT_INCLUDE +#error "I_AM_FORCED_COMPONENT_INCLUDE" +#endif + +#ifndef I_AM_FORCED_CPPPATH_INCLUDE +#error "I_AM_FORCED_CPPPATH_INCLUDE" +#endif + +#ifdef COMMENTED_MACRO +#error "COMMENTED_MACRO" +#endif + +int main() { +} +""" + ) + + tmpdir.mkdir("include").join("cpppath-include.h").write( + """ +#define I_AM_FORCED_CPPPATH_INCLUDE +""" + ) + component_dir = tmpdir.mkdir("lib").mkdir("component") + component_dir.join("component.h").write( + """ +#define I_AM_COMPONENT + +#ifndef I_AM_ONLY_SRC_FLAG +#error "I_AM_ONLY_SRC_FLAG" +#endif + +void dummy(void); + """ + ) + component_dir.join("component.cpp").write( + """ +#ifdef I_AM_ONLY_SRC_FLAG +#error "I_AM_ONLY_SRC_FLAG" +#endif + +void dummy(void ) {}; + """ + ) + component_dir.join("component-forced-include.h").write( + """ +#define I_AM_FORCED_COMPONENT_INCLUDE + """ + ) + + result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + validate_cliresult(result) + assert "post_prog_action is called" in result.output + build_output = result.output[result.output.find("Scanning dependencies...") :] + for flag in build_flags: + assert flag[1] in build_output, flag + + +def test_build_unflags(clirunner, validate_cliresult, tmpdir): + tmpdir.join("platformio.ini").write( + """ +[env:native] +platform = native +build_unflags = + -DTMP_MACRO_1=45 + -DTMP_MACRO_3=13 + -DTMP_MACRO_4 + -DNON_EXISTING_MACRO + -I. + -lunknownLib + -Os +build_flags = + -DTMP_MACRO_3=10 +extra_scripts = pre:extra.py +""" + ) + + tmpdir.join("extra.py").write( + """ +Import("env") +env.Append(CPPPATH="%s") +env.Append(CPPDEFINES="TMP_MACRO_1") +env.Append(CPPDEFINES=["TMP_MACRO_2"]) +env.Append(CPPDEFINES=[("TMP_MACRO_3", 13)]) +env.Append(CPPDEFINES=[("TMP_MACRO_4", 4)]) +env.Append(CCFLAGS=["-Os"]) +env.Append(LIBS=["unknownLib"]) + """ + % str(tmpdir) + ) + + tmpdir.mkdir("src").join("main.c").write( + """ +#ifndef TMP_MACRO_1 +#error "TMP_MACRO_1 should be defined" +#endif + +#ifndef TMP_MACRO_2 +#error "TMP_MACRO_2 should be defined" +#endif + +#if TMP_MACRO_3 != 10 +#error "TMP_MACRO_3 should be 10" +#endif + +#ifdef TMP_MACRO_4 +#error "TMP_MACRO_4 should not be defined" +#endif + +int main() { +} +""" + ) + + result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + validate_cliresult(result) + build_output = result.output[result.output.find("Scanning dependencies...") :] + assert "-DTMP_MACRO1" not in build_output + assert "-Os" not in build_output + assert str(tmpdir) not in build_output + + +def test_debug_default_build_flags(clirunner, validate_cliresult, tmpdir): + tmpdir.join("platformio.ini").write( + """ +[env:native] +platform = native +build_type = debug +""" + ) + + tmpdir.mkdir("src").join("main.c").write( + """ +int main() { +} +""" + ) + + result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + validate_cliresult(result) + build_output = result.output[result.output.find("Scanning dependencies...") :] + for line in build_output.split("\n"): + if line.startswith("gcc"): + assert all(line.count(flag) == 1 for flag in ("-Og", "-g2", "-ggdb2")) + assert all( + line.count("-%s%d" % (flag, level)) == 0 + for flag in ("O", "g", "ggdb") + for level in (0, 1, 3) + ) + assert "-Os" not in line + + +def test_debug_custom_build_flags(clirunner, validate_cliresult, tmpdir): + custom_debug_build_flags = ("-O3", "-g3", "-ggdb3") + + tmpdir.join("platformio.ini").write( + """ +[env:native] +platform = native +build_type = debug +debug_build_flags = %s + """ + % " ".join(custom_debug_build_flags) + ) + + tmpdir.mkdir("src").join("main.c").write( + """ +int main() { +} +""" + ) + + result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + validate_cliresult(result) + build_output = result.output[result.output.find("Scanning dependencies...") :] + for line in build_output.split("\n"): + if line.startswith("gcc"): + assert all(line.count(f) == 1 for f in custom_debug_build_flags) + assert all( + line.count("-%s%d" % (flag, level)) == 0 + for flag in ("O", "g", "ggdb") + for level in (0, 1, 2) + ) + assert all("-O%s" % optimization not in line for optimization in ("g", "s")) + + +def test_symlinked_libs(clirunner, validate_cliresult, tmp_path: Path): + external_pkg_dir = tmp_path / "External" + external_pkg_dir.mkdir() + (external_pkg_dir / "External.h").write_text( + """ +#define EXTERNAL 1 +""" + ) + (external_pkg_dir / "library.json").write_text( + """ +{ + "name": "External", + "version": "1.0.0" +} +""" + ) + + project_dir = tmp_path / "project" + src_dir = project_dir / "src" + src_dir.mkdir(parents=True) + (src_dir / "main.c").write_text( + """ +#include +# +#if !defined(EXTERNAL) +#error "EXTERNAL is not defined" +#endif + +int main() { +} +""" + ) + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +lib_deps = symlink://../External + """ + ) + result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir)]) + validate_cliresult(result) + + +def test_stringification(clirunner, validate_cliresult, tmp_path: Path): + project_dir = tmp_path / "project" + src_dir = project_dir / "src" + src_dir.mkdir(parents=True) + (src_dir / "main.c").write_text( + """ +#include +int main(void) { + printf("MACRO_1=<%s>\\n", MACRO_1); + printf("MACRO_2=<%s>\\n", MACRO_2); + printf("MACRO_3=<%s>\\n", MACRO_3); + printf("MACRO_4=<%s>\\n", MACRO_4); + return(0); +} +""" + ) + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +extra_scripts = script.py +build_flags = + '-DMACRO_1="Hello World!"' + '-DMACRO_2="Text is \\\\"Quoted\\\\""' + """ + ) + (project_dir / "script.py").write_text( + """ +Import("projenv") + +projenv.Append(CPPDEFINES=[ + ("MACRO_3", projenv.StringifyMacro('Hello "World"! Isn\\'t true?')), + ("MACRO_4", projenv.StringifyMacro("Special chars: ',(,),[,],:")) +]) + """ + ) + result = clirunner.invoke( + cmd_run, ["--project-dir", str(project_dir), "-t", "exec"] + ) + validate_cliresult(result) + assert "MACRO_1=" in result.output + assert 'MACRO_2=' in result.output + assert 'MACRO_3=' in result.output + assert "MACRO_4=" in result.output + + +def test_ldf(clirunner, validate_cliresult, tmp_path: Path): + project_dir = tmp_path / "project" + + # libs + lib_dir = project_dir / "lib" + a_lib_dir = lib_dir / "a" + a_lib_dir.mkdir(parents=True) + (a_lib_dir / "a.h").write_text( + """ +#include +""" + ) + # b + b_lib_dir = lib_dir / "b" + b_lib_dir.mkdir(parents=True) + (b_lib_dir / "some_from_b.h").write_text("") + # c + c_lib_dir = lib_dir / "c" + c_lib_dir.mkdir(parents=True) + (c_lib_dir / "parse_c_by_name.h").write_text( + """ +void some_func(); + """ + ) + (c_lib_dir / "parse_c_by_name.c").write_text( + """ +#include +#include + +void some_func() { +} + """ + ) + (c_lib_dir / "some.c").write_text( + """ +#include + """ + ) + # d + d_lib_dir = lib_dir / "d" + d_lib_dir.mkdir(parents=True) + (d_lib_dir / "d.h").write_text("") + + # project + src_dir = project_dir / "src" + src_dir.mkdir(parents=True) + (src_dir / "main.h").write_text( + """ +#include +#include +""" + ) + (src_dir / "main.c").write_text( + """ +#include + +int main() { +} +""" + ) + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native + """ + ) + result = clirunner.invoke(cmd_run, ["--project-dir", str(project_dir)]) + validate_cliresult(result) diff --git a/tests/commands/test_settings.py b/tests/commands/test_settings.py new file mode 100644 index 0000000..e846ecc --- /dev/null +++ b/tests/commands/test_settings.py @@ -0,0 +1,24 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import app +from platformio.commands.settings import cli + + +def test_settings_check(clirunner, validate_cliresult): + result = clirunner.invoke(cli, ["get"]) + validate_cliresult(result) + assert result.output + for item in app.DEFAULT_SETTINGS.items(): + assert item[0] in result.output diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py new file mode 100644 index 0000000..29ff20a --- /dev/null +++ b/tests/commands/test_test.py @@ -0,0 +1,660 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +from platformio import proc +from platformio.fs import load_json +from platformio.test.cli import cli as pio_test_cmd + + +def test_calculator_example(tmp_path: Path): + junit_output_path = tmp_path / "junit.xml" + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "calculator"), str(project_dir) + ) + result = proc.exec_command( + [ + "platformio", + "test", + "-d", + str(project_dir), + "-e", + "uno", + "-e", + "native", + "--junit-output-path", + str(junit_output_path), + ] + ) + assert result["returncode"] != 0 + # pylint: disable=unsupported-membership-test + assert all( + s in (result["err"] + result["out"]) for s in ("ERRORED", "PASSED", "FAILED") + ), result["out"] + + # test JUnit output + junit_testsuites = ET.parse(junit_output_path).getroot() + assert int(junit_testsuites.get("tests")) == 11 + assert int(junit_testsuites.get("errors")) == 2 + assert int(junit_testsuites.get("failures")) == 1 + assert len(junit_testsuites.findall("testsuite")) == 6 + junit_errored_testcase = junit_testsuites.find( + ".//testcase[@name='uno:test_embedded']" + ) + assert junit_errored_testcase.get("status") == "ERRORED" + assert junit_errored_testcase.find("error").get("type") == "UnitTestSuiteError" + junit_failed_testcase = junit_testsuites.find( + ".//testsuite[@name='native:test_desktop']" + "/testcase[@name='test_calculator_division']" + ) + assert junit_failed_testcase.get("status") == "FAILED" + assert junit_failed_testcase.find("failure").get("message") == "Expected 32 Was 33" + + +def test_list_tests(clirunner, validate_cliresult, tmp_path: Path): + json_output_path = tmp_path / "report.json" + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "calculator"), str(project_dir) + ) + result = clirunner.invoke( + pio_test_cmd, + [ + "-d", + str(project_dir), + "--list-tests", + "--json-output-path", + str(json_output_path), + ], + ) + validate_cliresult(result) + # test JSON + json_report = load_json(str(json_output_path)) + assert json_report["testcase_nums"] == 0 + assert json_report["failure_nums"] == 0 + assert json_report["skipped_nums"] == 0 + assert len(json_report["test_suites"]) == 6 + + +def test_group_and_custom_runner(clirunner, validate_cliresult, tmp_path: Path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +test_framework = custom +""" + ) + test_dir = project_dir / "test" + + # non-test folder, does not start with "test_" + disabled_dir = test_dir / "disabled" + disabled_dir.mkdir(parents=True) + (disabled_dir / "main.c").write_text( + """ +#include + +int main() { + printf("Disabled test suite\\n") +} + """ + ) + + # root + (test_dir / "my_extra.h").write_text( + """ +#ifndef MY_EXTRA_H +#define MY_EXTRA_H + +#include + +void my_extra_fun(void); +#endif +""" + ) + (test_dir / "my_extra.c").write_text( + """ +#include "my_extra.h" + +void my_extra_fun(void) { + printf("Called from my_extra_fun\\n"); +} +""" + ) + + # test group + test_group = test_dir / "group" + test_group.mkdir(parents=True) + (test_group / "test_custom_runner.py").write_text( + """ +import click + +from platformio.test.runners.unity import UnityTestRunner + +class CustomTestRunner(UnityTestRunner): + def teardown(self): + click.echo("CustomTestRunner::TearDown called") +""" + ) + + # test suite + test_suite_dir = test_group / "test_nested" + test_include_dir = test_suite_dir / "include" + test_include_dir.mkdir(parents=True) + (test_include_dir / "my_nested.h").write_text( + """ +#define TEST_ONE 1 +""" + ) + (test_suite_dir / "main.c").write_text( + """ +#include +#include +#include + +void setUp(){ + my_extra_fun(); +} + +void tearDown(void) { + // clean stuff up here +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, TEST_ONE); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); +} + """ + ) + result = clirunner.invoke( + pio_test_cmd, + ["-d", str(project_dir), "-e", "native", "--verbose"], + ) + validate_cliresult(result) + assert "1 Tests 0 Failures 0 Ignored" in result.output + assert "Called from my_extra_fun" in result.output + assert "CustomTestRunner::TearDown called" in result.output + assert "Disabled test suite" not in result.output + + +def test_crashed_program(clirunner, tmpdir): + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:native] +platform = native +""" + ) + test_dir = project_dir.mkdir("test") + test_dir.join("test_main.c").write( + """ +#include +#include + +void setUp(){ + printf("setUp called"); +} +void tearDown(){ + printf("tearDown called"); +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, 1); +} + +int main(int argc, char *argv[]) { + printf("Address boundary error is %s", argv[-1]); + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); + return 0; +} +""" + ) + result = clirunner.invoke( + pio_test_cmd, + ["-d", str(project_dir), "-e", "native"], + ) + assert result.exit_code != 0 + assert any( + s in result.output for s in ("Program received signal", "Program errored with") + ) + + +@pytest.mark.skipif( + sys.platform != "darwin", reason="runs only on macOS (issue with SimAVR)" +) +def test_custom_testing_command(clirunner, validate_cliresult, tmp_path: Path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text( + """ +[env:uno] +platform = atmelavr +framework = arduino +board = uno + +platform_packages = + platformio/tool-simavr @ ^1 +test_speed = 9600 +test_testing_command = + ${platformio.packages_dir}/tool-simavr/bin/simavr + -m + atmega328p + -f + 16000000L + ${platformio.build_dir}/${this.__env__}/firmware.elf +""" + ) + test_dir = project_dir / "test" / "test_dummy" + test_dir.mkdir(parents=True) + (test_dir / "test_main.cpp").write_text( + """ +#include +#include + +void setUp(void) { + // set stuff up here +} + +void tearDown(void) { + // clean stuff up here +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, 1); +} + +void setup() { + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); +} + +void loop() { + delay(1000); +} +""" + ) + result = clirunner.invoke( + pio_test_cmd, + ["-d", str(project_dir), "--without-uploading"], + ) + validate_cliresult(result) + assert "dummy_test" in result.output + + +def test_unity_setup_teardown(clirunner, validate_cliresult, tmpdir): + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:native] +platform = native +""" + ) + test_dir = project_dir.mkdir("test") + test_dir.join("test_main.h").write( + """ +#include +#include + """ + ) + test_dir.join("test_main.c").write( + """ +#include "test_main.h" + +void setUp(){ + printf("setUp called"); +} +void tearDown(){ + printf("tearDown called"); +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, 1); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); +} +""" + ) + result = clirunner.invoke( + pio_test_cmd, + ["-d", str(project_dir), "-e", "native"], + ) + validate_cliresult(result) + assert all(f in result.output for f in ("setUp called", "tearDown called")) + + +def test_unity_custom_config(clirunner, validate_cliresult, tmp_path: Path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +""" + ) + test_dir = project_dir / "test" / "native" / "test_component" + test_dir.mkdir(parents=True) + (test_dir.parent / "unity_config.h").write_text( + """ +#include + +#define CUSTOM_UNITY_CONFIG + +#define UNITY_OUTPUT_CHAR(c) putchar(c) +#define UNITY_OUTPUT_FLUSH() fflush(stdout) +""" + ) + (test_dir / "test_main.c").write_text( + """ +#include +#include + +void setUp(){ +#ifdef CUSTOM_UNITY_CONFIG + printf("Found custom unity_config.h\\n"); +#endif +} +void tearDown(){ +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, 1); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); +} +""" + ) + result = clirunner.invoke( + pio_test_cmd, + ["-d", str(project_dir), "-e", "native", "--verbose"], + ) + validate_cliresult(result) + assert all(f in result.output for f in ("Found custom unity_config", "dummy_test")) + + +def test_legacy_unity_custom_transport(clirunner, validate_cliresult, tmpdir): + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:embedded] +platform = ststm32 +framework = stm32cube +board = nucleo_f401re +test_transport = custom +""" + ) + + test_dir = project_dir.mkdir("test") + test_dir.join("test_main.c").write( + """ +#include + +void setUp(void) { + // set stuff up here +} + +void tearDown(void) { + // clean stuff up here +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, 1); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); +} +""" + ) + test_dir.join("unittest_transport.h").write( + """ +#ifdef __cplusplus +extern "C" { +#endif + +void unittest_uart_begin(){} +void unittest_uart_putchar(char c){} +void unittest_uart_flush(){} +void unittest_uart_end(){} + +#ifdef __cplusplus +} +#endif +""" + ) + result = clirunner.invoke( + pio_test_cmd, + [ + "-d", + str(project_dir), + "--without-testing", + "--without-uploading", + ], + ) + validate_cliresult(result) + + +@pytest.mark.skipif( + sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", + reason="skip Github Actions on Windows (MinGW issue)", +) +def test_doctest_framework(clirunner, tmp_path: Path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text( + """ +[env:native] +platform = native +test_framework = doctest +""" + ) + test_dir = project_dir / "test" / "test_dummy" + test_dir.mkdir(parents=True) + (test_dir / "test_main.cpp").write_text( + """ +#define DOCTEST_CONFIG_IMPLEMENT +#include + +TEST_CASE("[math] basic stuff") +{ + CHECK(6 > 5); + CHECK(6 > 7); +} + +TEST_CASE("should be skipped " * doctest::skip()) +{ + CHECK(2 > 5); +} + +TEST_CASE("vectors can be sized and resized") +{ + std::vector v(5); + + REQUIRE(v.size() == 5); + REQUIRE(v.capacity() >= 5); + + SUBCASE("adding to the vector increases it's size") + { + v.push_back(1); + + CHECK(v.size() == 6); + CHECK(v.capacity() >= 6); + } + SUBCASE("reserving increases just the capacity") + { + v.reserve(6); + + CHECK(v.size() == 5); + CHECK(v.capacity() >= 6); + } +} + +TEST_CASE("WARN level of asserts don't fail the test case") +{ + WARN(0); + WARN_FALSE(1); + WARN_EQ(1, 0); +} + +TEST_SUITE("scoped test suite") +{ + TEST_CASE("part of scoped") + { + FAIL("Error message"); + } + + TEST_CASE("part of scoped 2") + { + FAIL(""); + } +} + +int main(int argc, char **argv) +{ + doctest::Context context; + context.setOption("success", true); + context.setOption("no-exitcode", true); + context.applyCommandLine(argc, argv); + return context.run(); +} +""" + ) + junit_output_path = tmp_path / "junit.xml" + result = clirunner.invoke( + pio_test_cmd, + [ + "-d", + str(project_dir), + "--junit-output-path", + str(junit_output_path), + ], + ) + assert result.exit_code != 0 + # test JUnit output + junit_testsuites = ET.parse(junit_output_path).getroot() + assert int(junit_testsuites.get("tests")) == 8 + assert int(junit_testsuites.get("errors")) == 0 + assert int(junit_testsuites.get("failures")) == 3 + assert len(junit_testsuites.findall("testsuite")) == 1 + junit_failed_testcase = junit_testsuites.find( + ".//testcase[@name='scoped test suite/part of scoped']" + ) + assert junit_failed_testcase.get("status") == "FAILED" + assert junit_failed_testcase.find("failure").get("message") == "Error message" + assert "TEST SUITE: scoped test suite" in junit_failed_testcase.find("failure").text + + # test program arguments + json_output_path = tmp_path / "report.json" + result = clirunner.invoke( + pio_test_cmd, + [ + "-d", + str(project_dir), + "--json-output-path", + str(json_output_path), + "-a", + "-aa=1", # fail after the 1 error + ], + ) + assert result.exit_code != 0 + assert "1 test cases" in result.output + # test JSON + json_report = load_json(str(json_output_path)) + assert json_report["testcase_nums"] == 1 + assert json_report["failure_nums"] == 1 + + +@pytest.mark.skipif( + sys.platform == "win32" and os.environ.get("GITHUB_ACTIONS") == "true", + reason="skip Github Actions on Windows (MinGW issue)", +) +def test_googletest_framework(clirunner, tmp_path: Path): + project_dir = tmp_path / "project" + shutil.copytree( + os.path.join("examples", "unit-testing", "googletest"), str(project_dir) + ) + junit_output_path = tmp_path / "junit.xml" + result = clirunner.invoke( + pio_test_cmd, + [ + "-d", + str(project_dir), + "-e", + "native", + "--junit-output-path", + str(junit_output_path), + ], + ) + assert result.exit_code != 0 + # test JUnit output + junit_testsuites = ET.parse(junit_output_path).getroot() + assert int(junit_testsuites.get("tests")) == 4 + assert int(junit_testsuites.get("errors")) == 0 + assert int(junit_testsuites.get("failures")) == 1 + assert len(junit_testsuites.findall("testsuite")) == 4 + junit_failed_testcase = junit_testsuites.find(".//testcase[@name='FooTest.Bar']") + assert junit_failed_testcase.get("status") == "FAILED" + assert "test_main.cpp" in junit_failed_testcase.get("file") + assert junit_failed_testcase.get("line") == "26" + assert junit_failed_testcase.find("failure").get("message") == "Failure" + assert "Expected equality" in junit_failed_testcase.find("failure").text + + # test program arguments + json_output_path = tmp_path / "report.json" + result = clirunner.invoke( + pio_test_cmd, + [ + "-d", + str(project_dir), + "-e", + "native", + "--json-output-path", + str(json_output_path), + "-a", + "--gtest_filter=-FooTest.Bar", + ], + ) + assert result.exit_code == 0 + # test JSON + json_report = load_json(str(json_output_path)) + assert json_report["testcase_nums"] == 3 + assert json_report["failure_nums"] == 0 + assert json_report["skipped_nums"] == 1 + assert len(json_report["test_suites"]) == 4 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..eedf54c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,150 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import email +import functools +import imaplib +import os +import time + +import pytest +from click.testing import CliRunner + +from platformio import http +from platformio.package.meta import PackageSpec, PackageType +from platformio.registry.client import RegistryClient + + +def pytest_configure(config): + config.addinivalue_line("markers", "skip_ci: mark a test that will not run in CI") + + +@pytest.fixture(scope="session") +def validate_cliresult(): + def decorator(result): + assert result.exit_code == 0, "{} => {}".format(result.exception, result.output) + assert not result.exception, "{} => {}".format(result.exception, result.output) + + return decorator + + +@pytest.fixture(scope="session") +def clirunner(request, tmpdir_factory): + cache_dir = tmpdir_factory.mktemp(".cache") + backup_env_vars = { + "PLATFORMIO_CACHE_DIR": {"new": str(cache_dir)}, + "PLATFORMIO_WORKSPACE_DIR": {"new": None}, + } + for key, item in backup_env_vars.items(): + # pylint: disable=unnecessary-dict-index-lookup + backup_env_vars[key]["old"] = os.environ.get(key) + if item["new"] is not None: + os.environ[key] = item["new"] + elif key in os.environ: + del os.environ[key] + + def fin(): + for key, item in backup_env_vars.items(): + if item["old"] is not None: + os.environ[key] = item["old"] + elif key in os.environ: + del os.environ[key] + + request.addfinalizer(fin) + + return CliRunner() + + +def _isolated_pio_core(request, tmpdir_factory): + core_dir = tmpdir_factory.mktemp(".platformio") + os.environ["PLATFORMIO_CORE_DIR"] = str(core_dir) + + def fin(): + if "PLATFORMIO_CORE_DIR" in os.environ: + del os.environ["PLATFORMIO_CORE_DIR"] + + request.addfinalizer(fin) + return core_dir + + +@pytest.fixture(scope="module") +def isolated_pio_core(request, tmpdir_factory): + return _isolated_pio_core(request, tmpdir_factory) + + +@pytest.fixture(scope="function") +def func_isolated_pio_core(request, tmpdir_factory): + return _isolated_pio_core(request, tmpdir_factory) + + +@pytest.fixture(scope="function") +def without_internet(monkeypatch): + monkeypatch.setattr(http, "_internet_on", lambda: False) + + +@pytest.fixture +def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals + def _receive_email(from_who): + test_email = os.environ["TEST_EMAIL_LOGIN"] + test_password = os.environ["TEST_EMAIL_PASSWORD"] + imap_server = os.environ["TEST_EMAIL_IMAP_SERVER"] + + def get_body(msg): + if msg.is_multipart(): + return get_body(msg.get_payload(0)) + return msg.get_payload(None, True) + + result = None + start_time = time.time() + while not result: + time.sleep(5) + server = imaplib.IMAP4_SSL(imap_server) + server.login(test_email, test_password) + server.select("INBOX") + _, mails = server.search(None, "ALL") + for index in mails[0].split(): + status, data = server.fetch(index, "(RFC822)") + if status != "OK" or not data or not isinstance(data[0], tuple): + continue + msg = email.message_from_string( + data[0][1].decode("ASCII", errors="surrogateescape") + ) + if from_who not in msg.get("To"): + continue + if "gmail" in imap_server: + server.store(index, "+X-GM-LABELS", "\\Trash") + server.store(index, "+FLAGS", "\\Deleted") + server.expunge() + result = get_body(msg).decode() + if time.time() - start_time > 120: + break + server.close() + server.logout() + return result + + return _receive_email + + +@pytest.fixture(scope="session") +def get_pkg_latest_version(): + @functools.lru_cache() + def wrap(spec, pkg_type=None): + if not isinstance(spec, PackageSpec): + spec = PackageSpec(spec) + pkg_type = pkg_type or PackageType.LIBRARY + client = RegistryClient() + pkg = client.get_package(pkg_type, spec.owner, spec.name) + return pkg["version"]["name"] + + return wrap diff --git a/tests/misc/__init__.py b/tests/misc/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/tests/misc/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/misc/ino2cpp/__init__.py b/tests/misc/ino2cpp/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/tests/misc/ino2cpp/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/misc/ino2cpp/examples/basic/basic.ino b/tests/misc/ino2cpp/examples/basic/basic.ino new file mode 100644 index 0000000..fcec8ed --- /dev/null +++ b/tests/misc/ino2cpp/examples/basic/basic.ino @@ -0,0 +1,60 @@ +#define SQR(a) \ +( a * a ) + +typedef struct Item item; +struct Item { + byte foo[50]; + int bar; + void (*noob)(item*); +}; + +// test callback +class Foo { + + public: + Foo(void (*function)()) { + #warning "Line number is 16" + } + + bool childFunc() { + + } + +}; + +Foo foo(&fooCallback); + +// + +template T Add(T n1, T n2) { + return n1 + n2; +} + +void setup() { + struct Item item1; + myFunction(&item1); +} + +void loop() { + +} + +void myFunction(struct Item *item) { + +} + +#warning "Line number is 46" + +void fooCallback(){ + +} + +extern "C" { +void some_extern(const char *fmt, ...); +}; + +void some_extern(const char *fmt, ...) { + +} + +// юнікод diff --git a/tests/misc/ino2cpp/examples/multifiles/bar.ino b/tests/misc/ino2cpp/examples/multifiles/bar.ino new file mode 100644 index 0000000..6040586 --- /dev/null +++ b/tests/misc/ino2cpp/examples/multifiles/bar.ino @@ -0,0 +1,4 @@ +unsigned int barFunc () // my comment +{ + return 0; +} \ No newline at end of file diff --git a/tests/misc/ino2cpp/examples/multifiles/foo.pde b/tests/misc/ino2cpp/examples/multifiles/foo.pde new file mode 100644 index 0000000..9b2913c --- /dev/null +++ b/tests/misc/ino2cpp/examples/multifiles/foo.pde @@ -0,0 +1,13 @@ +char buf[5]; + +void setup() { + fooFunc(); +} + +void loop() { + +} + +char* fooFunc() { + return buf; +} diff --git a/tests/misc/ino2cpp/examples/strmultilines/main.ino b/tests/misc/ino2cpp/examples/strmultilines/main.ino new file mode 100644 index 0000000..c5df8f3 --- /dev/null +++ b/tests/misc/ino2cpp/examples/strmultilines/main.ino @@ -0,0 +1,79 @@ +const char headerEndP[] PROGMEM = +"\ +\ +\ +"; + +void setup() { + +} + +const char javaScriptPinControlP[] PROGMEM = +"
\ +
\ +"; + +#warning "Line 75" + +void loop() { + +} \ No newline at end of file diff --git a/tests/misc/ino2cpp/test_ino2cpp.py b/tests/misc/ino2cpp/test_ino2cpp.py new file mode 100644 index 0000000..72e3bcd --- /dev/null +++ b/tests/misc/ino2cpp/test_ino2cpp.py @@ -0,0 +1,48 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import listdir +from os.path import dirname, isdir, join, normpath + +from platformio.commands.ci import cli as cmd_ci + +EXAMPLES_DIR = normpath(join(dirname(__file__), "examples")) + + +def pytest_generate_tests(metafunc): + if "piotest_dir" not in metafunc.fixturenames: + return + test_dirs = [] + for name in listdir(EXAMPLES_DIR): + if isdir(join(EXAMPLES_DIR, name)): + test_dirs.append(join(EXAMPLES_DIR, name)) + test_dirs.sort() + metafunc.parametrize("piotest_dir", test_dirs) + + +def test_example(clirunner, validate_cliresult, piotest_dir): + result = clirunner.invoke(cmd_ci, [piotest_dir, "-b", "uno"]) + validate_cliresult(result) + + +def test_warning_line(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_ci, [join(EXAMPLES_DIR, "basic"), "-b", "uno"]) + validate_cliresult(result) + assert 'basic.ino:16:14: warning: #warning "Line number is 16"' in result.output + assert 'basic.ino:46:2: warning: #warning "Line number is 46"' in result.output + result = clirunner.invoke( + cmd_ci, [join(EXAMPLES_DIR, "strmultilines"), "-b", "uno"] + ) + validate_cliresult(result) + assert 'main.ino:75:2: warning: #warning "Line 75"' in result.output diff --git a/tests/misc/test_maintenance.py b/tests/misc/test_maintenance.py new file mode 100644 index 0000000..8a403ae --- /dev/null +++ b/tests/misc/test_maintenance.py @@ -0,0 +1,50 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +from time import time + +from platformio import app, maintenance +from platformio.__main__ import cli as cli_pio +from platformio.commands import upgrade as cmd_upgrade + + +def test_check_pio_upgrade(clirunner, isolated_pio_core, validate_cliresult): + def _patch_pio_version(version): + maintenance.__version__ = version + cmd_upgrade.VERSION = version.split(".", 3) + + interval = int(app.get_setting("check_platformio_interval")) * 3600 * 24 + last_check = {"platformio_upgrade": time() - interval - 1} + origin_version = maintenance.__version__ + + # check development version + _patch_pio_version("3.0.0-a1") + app.set_state_item("last_check", last_check) + result = clirunner.invoke(cli_pio, ["platform", "list"]) + validate_cliresult(result) + assert "There is a new version" in result.output + assert "Please upgrade" in result.output + + # check stable version + _patch_pio_version("2.11.0") + app.set_state_item("last_check", last_check) + result = clirunner.invoke(cli_pio, ["platform", "list"]) + validate_cliresult(result) + assert "There is a new version" in result.output + assert "Please upgrade" in result.output + + # restore original version + _patch_pio_version(origin_version) diff --git a/tests/misc/test_misc.py b/tests/misc/test_misc.py new file mode 100644 index 0000000..d37cb71 --- /dev/null +++ b/tests/misc/test_misc.py @@ -0,0 +1,48 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import pytest +import requests + +from platformio import __check_internet_hosts__, http, proc +from platformio.registry.client import RegistryClient + + +def test_platformio_cli(): + result = proc.exec_command(["pio", "--help"]) + assert result["returncode"] == 0 + # pylint: disable=unsupported-membership-test + assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result["out"] + + +def test_ping_internet_ips(): + for host in __check_internet_hosts__: + requests.get("http://%s" % host, allow_redirects=False, timeout=2) + + +def test_api_internet_offline(without_internet, isolated_pio_core): + regclient = RegistryClient() + with pytest.raises(http.InternetConnectionError): + regclient.fetch_json_data("get", "/v3/search") + + +def test_api_cache(monkeypatch, isolated_pio_core): + regclient = RegistryClient() + api_kwargs = {"method": "get", "path": "/v3/search", "x_cache_valid": "10s"} + result = regclient.fetch_json_data(**api_kwargs) + assert result and "total" in result + monkeypatch.setattr(http, "_internet_on", lambda: False) + assert regclient.fetch_json_data(**api_kwargs) == result diff --git a/tests/package/__init__.py b/tests/package/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/tests/package/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py new file mode 100644 index 0000000..c1de90b --- /dev/null +++ b/tests/package/test_manager.py @@ -0,0 +1,654 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import logging +import os +import time +from pathlib import Path +from random import random + +import pytest +import semantic_version + +from platformio import fs, util +from platformio.package.exception import ( + MissingPackageManifestError, + UnknownPackageError, +) +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.package.pack import PackagePacker + + +def test_download(isolated_pio_core): + url = "https://github.com/platformio/platformio-core/archive/v4.3.4.zip" + checksum = "69d59642cb91e64344f2cdc1d3b98c5cd57679b5f6db7accc7707bd4c5d9664a" + lm = LibraryPackageManager() + lm.set_log_level(logging.ERROR) + archive_path = lm.download(url, checksum) + assert fs.calculate_file_hashsum("sha256", archive_path) == checksum + lm.cleanup_expired_downloads(random()) + assert os.path.isfile(archive_path) + # test outdated downloads + lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1) + lm.cleanup_expired_downloads(random()) + assert not os.path.isfile(archive_path) + # check that key is deleted from DB + with open(lm.get_download_usagedb_path(), encoding="utf8") as fp: + assert os.path.basename(archive_path) not in fp.read() + + +def test_find_pkg_root(isolated_pio_core, tmpdir_factory): + # has manifest + pkg_dir = tmpdir_factory.mktemp("package-has-manifest") + root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() + root_dir.join("platform.json").write("") + pm = PlatformPackageManager() + found_dir = pm.find_pkg_root(str(pkg_dir), spec=None) + assert os.path.realpath(str(root_dir)) == os.path.realpath(found_dir) + + # does not have manifest + pkg_dir = tmpdir_factory.mktemp("package-does-not-have-manifest") + pkg_dir.join("nested").mkdir().join("folder").mkdir().join("readme.txt").write("") + pm = PlatformPackageManager() + with pytest.raises(MissingPackageManifestError): + pm.find_pkg_root(str(pkg_dir), spec=None) + + # library package without manifest, should find source root + pkg_dir = tmpdir_factory.mktemp("library-package-without-manifest") + root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() + root_dir.join("src").mkdir().join("main.cpp").write("") + root_dir.join("include").mkdir().join("main.h").write("") + assert os.path.realpath(str(root_dir)) == os.path.realpath( + LibraryPackageManager.find_library_root(str(pkg_dir)) + ) + + # library manager should create "library.json" + lm = LibraryPackageManager() + spec = PackageSpec("custom-name@1.0.0") + pkg_root = lm.find_pkg_root(str(pkg_dir), spec) + manifest_path = os.path.join(pkg_root, "library.json") + assert os.path.realpath(str(root_dir)) == os.path.realpath(pkg_root) + assert os.path.isfile(manifest_path) + manifest = lm.load_manifest(pkg_root) + assert manifest["name"] == "custom-name" + assert "0.0.0" in str(manifest["version"]) + + +def test_build_legacy_spec(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + pm = PlatformPackageManager(str(storage_dir)) + # test src manifest + pkg1_dir = storage_dir.join("pkg-1").mkdir() + pkg1_dir.join(".pio").mkdir().join(".piopkgmanager.json").write( + """ +{ + "name": "StreamSpy-0.0.1.tar", + "url": "https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + "requirements": null +} +""" + ) + assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec( + name="StreamSpy-0.0.1.tar", + uri="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + ) + + # without src manifest + pkg2_dir = storage_dir.join("pkg-2").mkdir() + pkg2_dir.join("main.cpp").write("") + with pytest.raises(MissingPackageManifestError): + pm.build_legacy_spec(str(pkg2_dir)) + + # with package manifest + pkg3_dir = storage_dir.join("pkg-3").mkdir() + pkg3_dir.join("platform.json").write('{"name": "pkg3", "version": "1.2.0"}') + assert pm.build_legacy_spec(str(pkg3_dir)) == PackageSpec(name="pkg3") + + +def test_build_metadata(isolated_pio_core, tmpdir_factory): + pm = PlatformPackageManager() + vcs_revision = "a2ebfd7c0f" + pkg_dir = tmpdir_factory.mktemp("package") + + # test package without manifest + with pytest.raises(MissingPackageManifestError): + pm.load_manifest(str(pkg_dir)) + with pytest.raises(MissingPackageManifestError): + pm.build_metadata(str(pkg_dir), PackageSpec("MyLib")) + + # with manifest + pkg_dir.join("platform.json").write( + '{"name": "Dev-Platform", "version": "1.2.3-alpha.1"}' + ) + metadata = pm.build_metadata(str(pkg_dir), PackageSpec("owner/platform-name")) + assert metadata.name == "Dev-Platform" + assert str(metadata.version) == "1.2.3-alpha.1" + + # with vcs + metadata = pm.build_metadata( + str(pkg_dir), PackageSpec("owner/platform-name"), vcs_revision + ) + assert str(metadata.version) == ("1.2.3-alpha.1+sha." + vcs_revision) + assert metadata.version.build[1] == vcs_revision + + +def test_install_from_uri(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + + # install from local directory + src_dir = tmp_dir.join("local-lib-dir").mkdir() + src_dir.join("main.cpp").write("") + spec = PackageSpec("file://%s" % src_dir) + pkg = lm.install(spec) + assert os.path.isfile(os.path.join(pkg.path, "main.cpp")) + manifest = lm.load_manifest(pkg) + assert manifest["name"] == "local-lib-dir" + assert manifest["version"].startswith("0.0.0+") + assert spec == pkg.metadata.spec + + # install from local archive + src_dir = tmp_dir.join("archive-src").mkdir() + root_dir = src_dir.mkdir("root") + root_dir.mkdir("src").join("main.cpp").write("#include ") + root_dir.join("library.json").write( + '{"name": "manifest-lib-name", "version": "2.0.0"}' + ) + tarball_path = PackagePacker(str(src_dir)).pack(str(tmp_dir)) + spec = PackageSpec("file://%s" % tarball_path) + pkg = lm.install(spec) + assert os.path.isfile(os.path.join(pkg.path, "src", "main.cpp")) + assert pkg == lm.get_package(spec) + assert spec == pkg.metadata.spec + + # install from registry + src_dir = tmp_dir.join("registry-1").mkdir() + src_dir.join("library.properties").write( + """ +name = wifilib +version = 5.2.7 +""" + ) + spec = PackageSpec("company/wifilib @ ^5") + pkg = lm.install_from_uri("file://%s" % src_dir, spec) + assert str(pkg.metadata.version) == "5.2.7" + + # check package folder names + lm.memcache_reset() + assert ["local-lib-dir", "manifest-lib-name", "wifilib"] == [ + os.path.basename(pkg.path) for pkg in lm.get_installed() + ] + + +def test_install_from_registry(isolated_pio_core, tmpdir_factory): + # Libraries + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) + lm.set_log_level(logging.ERROR) + # library with dependencies + lm.install("AsyncMqttClient-esphome @ 0.8.6") + assert len(lm.get_installed()) == 3 + pkg = lm.get_package("AsyncTCP-esphome") + assert pkg.metadata.spec.owner == "esphome" + assert not lm.get_package("non-existing-package") + # mbed library + assert lm.install("wolfSSL") + assert len(lm.get_installed()) == 4 + # case sensitive author name + assert lm.install("DallasTemperature") + assert lm.get_package("OneWire").metadata.version.major >= 2 + assert len(lm.get_installed()) == 6 + + # test conflicted names + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage"))) + lm.set_log_level(logging.ERROR) + lm.install("z3t0/IRremote@2.6.1") + lm.install("mbed-yuhki50/IRremote") + assert len(lm.get_installed()) == 2 + + # Tools + tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) + tm.set_log_level(logging.ERROR) + pkg = tm.install("platformio/tool-stlink @ ~1.10400.0") + manifest = tm.load_manifest(pkg) + assert tm.is_system_compatible(manifest.get("system")) + assert util.get_systype() in manifest.get("system", []) + + # Test unknown + with pytest.raises(UnknownPackageError): + tm.install("unknown-package-tool @ 9.1.1") + with pytest.raises(UnknownPackageError): + tm.install("owner/unknown-package-tool") + + +def test_install_lib_depndencies(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + + src_dir = tmp_dir.join("lib-with-deps").mkdir() + root_dir = src_dir.mkdir("root") + root_dir.mkdir("src").join("main.cpp").write("#include ") + root_dir.join("library.json").write( + """ +{ + "name": "lib-with-deps", + "version": "2.0.0", + "dependencies": [ + { + "owner": "bblanchon", + "name": "ArduinoJson", + "version": "^6.16.1" + }, + { + "name": "external-repo", + "version": "https://github.com/milesburton/Arduino-Temperature-Control-Library.git#4a0ccc1" + } + ] +} +""" + ) + + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) + lm.set_log_level(logging.ERROR) + lm.install("file://%s" % str(src_dir)) + installed = lm.get_installed() + assert len(installed) == 4 + assert set(["external-repo", "ArduinoJson", "lib-with-deps", "OneWire"]) == set( + p.metadata.name for p in installed + ) + + +def test_install_force(isolated_pio_core, tmpdir_factory): + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) + lm.set_log_level(logging.ERROR) + # install #64 ArduinoJson + pkg = lm.install("64 @ ^5") + assert pkg.metadata.version.major == 5 + # try install the latest without specification + pkg = lm.install("64") + assert pkg.metadata.version.major == 5 + assert len(lm.get_installed()) == 1 + # re-install the latest + pkg = lm.install(64, force=True) + assert len(lm.get_installed()) == 1 + assert pkg.metadata.version.major > 5 + + +def test_symlink(tmp_path: Path): + external_pkg_dir = tmp_path / "External" + external_pkg_dir.mkdir() + (external_pkg_dir / "library.json").write_text( + """ +{ + "name": "External", + "version": "1.0.0" +} +""" + ) + + storage_dir = tmp_path / "storage" + installed_pkg_dir = storage_dir / "installed" + installed_pkg_dir.mkdir(parents=True) + (installed_pkg_dir / "library.json").write_text( + """ +{ + "name": "Installed", + "version": "1.0.0" +} +""" + ) + + spec = "CustomExternal=symlink://%s" % str(external_pkg_dir) + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + pkg = lm.install(spec) + assert os.path.isfile(str(storage_dir / "CustomExternal.pio-link")) + assert pkg.metadata.name == "External" + assert pkg.metadata.version.major == 1 + assert ["External", "Installed"] == [ + pkg.metadata.name for pkg in lm.get_installed() + ] + pkg = lm.get_package("External") + assert Path(pkg.path) == external_pkg_dir + assert pkg.metadata.spec.uri.startswith("symlink://") + assert lm.get_package(spec).metadata.spec.uri.startswith("symlink://") + + # try to update + lm.update(pkg) + + # uninstall + lm.uninstall("External") + assert ["Installed"] == [pkg.metadata.name for pkg in lm.get_installed()] + # ensure original package was not rmeoved + assert external_pkg_dir.is_dir() + + # install again, remove from a disk + assert lm.install("symlink://%s" % str(external_pkg_dir)) + assert os.path.isfile(str(storage_dir / "External.pio-link")) + assert ["External", "Installed"] == [ + pkg.metadata.name for pkg in lm.get_installed() + ] + fs.rmtree(str(external_pkg_dir)) + lm.memcache_reset() + assert ["Installed"] == [pkg.metadata.name for pkg in lm.get_installed()] + + +def test_scripts(isolated_pio_core, tmp_path: Path): + pkg_dir = tmp_path / "foo" + scripts_dir = pkg_dir / "scripts" + scripts_dir.mkdir(parents=True) + (scripts_dir / "script.py").write_text( + """ +import sys +from pathlib import Path + +action = "postinstall" if len(sys.argv) == 1 else sys.argv[1] +Path("%s.flag" % action).touch() + +if action == "preuninstall": + Path("../%s.flag" % action).touch() +""" + ) + (pkg_dir / "library.json").write_text( + """ +{ + "name": "foo", + "version": "1.0.0", + "scripts": { + "postinstall": "scripts/script.py", + "preuninstall2": ["scripts/script.py", "preuninstall"] + } +} +""" + ) + + storage_dir = tmp_path / "storage" + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + lm.install("file://%s" % str(pkg_dir)) + assert os.path.isfile(os.path.join(lm.get_package("foo").path, "postinstall.flag")) + lm.uninstall("foo") + (storage_dir / "preuninstall.flag").is_file() + + +def test_install_circular_dependencies(tmp_path: Path): + storage_dir = tmp_path / "storage" + # Foo + pkg_dir = storage_dir / "foo" + pkg_dir.mkdir(parents=True) + (pkg_dir / "library.json").write_text( + """ +{ + "name": "Foo", + "version": "1.0.0", + "dependencies": { + "Bar": "*" + } +} +""" + ) + # Bar + pkg_dir = storage_dir / "bar" + pkg_dir.mkdir(parents=True) + (pkg_dir / "library.json").write_text( + """ +{ + "name": "Bar", + "version": "1.0.0", + "dependencies": { + "Foo": "*" + } +} +""" + ) + + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + assert len(lm.get_installed()) == 2 + + # root library + pkg_dir = tmp_path / "root" + pkg_dir.mkdir(parents=True) + (pkg_dir / "library.json").write_text( + """ +{ + "name": "Root", + "version": "1.0.0", + "dependencies": { + "Foo": "^1.0.0", + "Bar": "^1.0.0" + } +} +""" + ) + lm.install("file://%s" % str(pkg_dir)) + + +def test_get_installed(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + pm = ToolPackageManager(str(storage_dir)) + + # VCS package + ( + storage_dir.join("pkg-vcs") + .mkdir() + .join(".git") + .mkdir() + .join(".piopm") + .write( + """ +{ + "name": "pkg-via-vcs", + "spec": { + "id": null, + "name": "pkg-via-vcs", + "owner": null, + "requirements": null, + "url": "git+https://github.com/username/repo.git" + }, + "type": "tool", + "version": "0.0.0+sha.1ea4d5e" +} +""" + ) + ) + + # package without metadata file + ( + storage_dir.join("foo@3.4.5") + .mkdir() + .join("package.json") + .write('{"name": "foo", "version": "3.4.5"}') + ) + + # package with metadata file + foo_dir = storage_dir.join("foo").mkdir() + foo_dir.join("package.json").write('{"name": "foo", "version": "3.6.0"}') + foo_dir.join(".piopm").write( + """ +{ + "name": "foo", + "spec": { + "name": "foo", + "owner": null, + "requirements": "^3" + }, + "type": "tool", + "version": "3.6.0" +} +""" + ) + + # test "system" + storage_dir.join("pkg-incompatible-system").mkdir().join("package.json").write( + '{"name": "check-system", "version": "4.0.0", "system": ["unknown"]}' + ) + storage_dir.join("pkg-compatible-system").mkdir().join("package.json").write( + '{"name": "check-system", "version": "3.0.0", "system": "%s"}' + % util.get_systype() + ) + + # invalid package + storage_dir.join("invalid-package").mkdir().join("library.json").write( + '{"name": "SomeLib", "version": "4.0.0"}' + ) + + installed = pm.get_installed() + assert len(installed) == 4 + assert set(["pkg-via-vcs", "foo", "check-system"]) == set( + p.metadata.name for p in installed + ) + assert str(pm.get_package("foo").metadata.version) == "3.6.0" + assert str(pm.get_package("check-system").metadata.version) == "3.0.0" + + +def test_uninstall(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + + # foo @ 1.0.0 + pkg_dir = tmp_dir.join("foo").mkdir() + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + foo_1_0_0_pkg = lm.install_from_uri("file://%s" % pkg_dir, "foo") + # foo @ 1.3.0 + pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') + lm.install_from_uri("file://%s" % pkg_dir, "foo") + # bar + pkg_dir = tmp_dir.join("bar").mkdir() + pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') + bar_pkg = lm.install("file://%s" % pkg_dir) + + assert len(lm.get_installed()) == 3 + assert os.path.isdir(os.path.join(str(storage_dir), "foo")) + assert os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) + + # check detaching + assert lm.uninstall("FOO") + assert len(lm.get_installed()) == 2 + assert os.path.isdir(os.path.join(str(storage_dir), "foo")) + assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) + + # uninstall the rest + assert lm.uninstall(foo_1_0_0_pkg.path) + assert lm.uninstall(bar_pkg) + + assert not lm.get_installed() + + # test uninstall dependencies + assert lm.install("AsyncMqttClient-esphome @ 0.8.4") + assert len(lm.get_installed()) == 3 + assert lm.uninstall("AsyncMqttClient-esphome", skip_dependencies=True) + assert len(lm.get_installed()) == 2 + + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + assert lm.install("AsyncMqttClient-esphome @ 0.8.4") + assert lm.uninstall("AsyncMqttClient-esphome") + assert not lm.get_installed() + + +def test_registry(isolated_pio_core): + lm = LibraryPackageManager() + lm.set_log_level(logging.ERROR) + + # reveal ID + assert lm.reveal_registry_package_id(PackageSpec(id=13)) == 13 + assert lm.reveal_registry_package_id(PackageSpec(name="OneWire")) == 1 + with pytest.raises(UnknownPackageError): + lm.reveal_registry_package_id(PackageSpec(name="/non-existing-package/")) + + # fetch package data + assert lm.fetch_registry_package(PackageSpec(id=1))["name"] == "OneWire" + assert lm.fetch_registry_package(PackageSpec(name="ArduinoJson"))["id"] == 64 + assert ( + lm.fetch_registry_package( + PackageSpec(id=13, owner="adafruit", name="Renamed library") + )["name"] + == "Adafruit GFX Library" + ) + with pytest.raises(UnknownPackageError): + lm.fetch_registry_package( + PackageSpec(owner="unknown<>owner", name="/non-existing-package/") + ) + with pytest.raises(UnknownPackageError): + lm.fetch_registry_package(PackageSpec(name="/non-existing-package/")) + + +def test_update_with_metadata(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + + # test non SemVer in registry + pkg = lm.install("adafruit/Adafruit NeoPixel @ <1.9") + outdated = lm.outdated(pkg) + assert str(outdated.current) == "1.8.7" + assert outdated.latest > semantic_version.Version("1.10.0") + + pkg = lm.install("ArduinoJson @ 5.10.1") + # test latest + outdated = lm.outdated(pkg) + assert str(outdated.current) == "5.10.1" + assert outdated.wanted is None + assert outdated.latest > outdated.current + assert outdated.latest > semantic_version.Version("5.99.99") + + # test wanted + outdated = lm.outdated(pkg, PackageSpec("ArduinoJson@~5")) + assert str(outdated.current) == "5.10.1" + assert str(outdated.wanted) == "5.13.4" + assert outdated.latest > semantic_version.Version("6.16.0") + + # update to the wanted 5.x + new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5")) + assert str(new_pkg.metadata.version) == "5.13.4" + # check that old version is removed + assert len(lm.get_installed()) == 2 + + # update to the latest + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + pkg = lm.update("ArduinoJson") + assert pkg.metadata.version == outdated.latest + + +def test_update_without_metadata(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + storage_dir.join("legacy-package").mkdir().join("library.json").write( + '{"name": "AsyncMqttClient-esphome", "version": "0.8"}' + ) + storage_dir.join("legacy-dep").mkdir().join("library.json").write( + '{"name": "AsyncTCP-esphome", "version": "1.1.1"}' + ) + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.get_package("AsyncMqttClient-esphome") + outdated = lm.outdated(pkg) + assert len(lm.get_installed()) == 2 + assert str(pkg.metadata.version) == "0.8.0" + assert outdated.latest > semantic_version.Version("0.8.0") + + # update + lm = LibraryPackageManager(str(storage_dir)) + lm.set_log_level(logging.ERROR) + new_pkg = lm.update(pkg) + assert len(lm.get_installed()) == 4 + assert new_pkg.metadata.spec.owner == "ottowinter" diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py new file mode 100644 index 0000000..bcc7e00 --- /dev/null +++ b/tests/package/test_manifest.py @@ -0,0 +1,905 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import tarfile + +import jsondiff +import pytest + +from platformio.compat import IS_WINDOWS +from platformio.package.manifest import parser +from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError + + +def test_library_json_parser(): + contents = """ +{ + "name": "TestPackage", + "keywords": "kw1, KW2, kw3, KW2, kw 4, kw_5, kw-6", + "headers": "include1.h, Include2.hpp", + "platforms": ["atmelavr", "espressif"], + "repository": { + "type": "git", + "url": "http://github.com/username/repo/" + }, + "url": "http://old.url.format", + "exclude": [".gitignore", "tests"], + "include": "mylib", + "build": { + "flags": ["-DHELLO"] + }, + "examples": ["examples/*/*.pde"], + "dependencies": { + "deps1": "1.2.0", + "deps2": "https://github.com/username/package.git", + "owner/deps3": "^2.1.3" + }, + "customField": "Custom Value" +} +""" + raw_data = parser.LibraryJsonManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + assert not jsondiff.diff( + raw_data, + { + "name": "TestPackage", + "platforms": ["atmelavr", "espressif8266"], + "repository": { + "type": "git", + "url": "https://github.com/username/repo.git", + }, + "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, + "keywords": ["kw1", "kw2", "kw3", "kw 4", "kw_5", "kw-6"], + "headers": ["include1.h", "Include2.hpp"], + "homepage": "http://old.url.format", + "build": {"flags": ["-DHELLO"]}, + "dependencies": [ + {"name": "deps1", "version": "1.2.0"}, + {"name": "deps2", "version": "https://github.com/username/package.git"}, + {"owner": "owner", "name": "deps3", "version": "^2.1.3"}, + ], + "customField": "Custom Value", + }, + ) + + contents = """ +{ + "keywords": ["sound", "audio", "music", "SD", "card", "playback"], + "headers": ["include 1.h", "include Space.hpp"], + "frameworks": "arduino", + "export": { + "exclude": "audio_samples" + }, + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + {"owner": "owner", "name": "deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"}, + {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} + ] +} +""" + raw_data = parser.LibraryJsonManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + assert not jsondiff.diff( + raw_data, + { + "keywords": ["sound", "audio", "music", "sd", "card", "playback"], + "headers": ["include 1.h", "include Space.hpp"], + "frameworks": ["arduino"], + "export": {"exclude": ["audio_samples"]}, + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + { + "owner": "owner", + "name": "deps2", + "version": "1.0.0", + "platforms": ["*"], + "frameworks": ["arduino", "espidf"], + }, + { + "name": "deps3", + "version": "1.0.0", + "platforms": ["ststm32", "sifive"], + }, + ], + }, + ) + + raw_data = parser.LibraryJsonManifestParser( + '{"dependencies": ["dep1", "dep2", "owner/dep3@1.2.3"]}' + ).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + assert not jsondiff.diff( + raw_data, + { + "dependencies": [ + {"name": "dep1"}, + {"name": "dep2"}, + {"name": "owner/dep3@1.2.3"}, + ], + }, + ) + + # broken dependencies + with pytest.raises(parser.ManifestParserError): + parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]}) + + +def test_module_json_parser(): + contents = """ +{ + "author": "Name Surname ", + "description": "This is Yotta library", + "homepage": "https://yottabuild.org", + "keywords": [ + "mbed", + "Yotta" + ], + "licenses": [ + { + "type": "Apache-2.0", + "url": "https://spdx.org/licenses/Apache-2.0" + } + ], + "name": "YottaLibrary", + "repository": { + "type": "git", + "url": "git@github.com:username/repo.git" + }, + "version": "1.2.3", + "dependencies": { + "usefulmodule": "^1.2.3", + "simplelog": "ARMmbed/simplelog#~0.0.1" + }, + "customField": "Custom Value" +} +""" + + raw_data = parser.ModuleJsonManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + assert not jsondiff.diff( + raw_data, + { + "name": "YottaLibrary", + "description": "This is Yotta library", + "homepage": "https://yottabuild.org", + "keywords": ["mbed", "yotta"], + "license": "Apache-2.0", + "platforms": ["*"], + "frameworks": ["mbed"], + "export": {"exclude": ["tests", "test", "*.doxyfile", "*.pdf"]}, + "authors": [{"email": "name@surname.com", "name": "Name Surname"}], + "version": "1.2.3", + "repository": {"type": "git", "url": "git@github.com:username/repo.git"}, + "dependencies": [ + { + "name": "simplelog", + "version": "ARMmbed/simplelog#~0.0.1", + "frameworks": ["mbed"], + }, + {"name": "usefulmodule", "version": "^1.2.3", "frameworks": ["mbed"]}, + ], + "customField": "Custom Value", + }, + ) + + +def test_library_properties_parser(): + # Base + contents = """ +name=TestPackage +version=1.2.3 +author=SomeAuthor , Maintainer Author (nickname) +maintainer=Maintainer Author (nickname) +sentence=This is Arduino library +category=Signal Input/Output +customField=Custom Value +depends=First Library (=2.0.0), Second Library (>=1.2.0), Third +ignore_empty_field= +includes=Arduino.h, Arduino Space.hpp +""" + raw_data = parser.LibraryPropertiesManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + assert not jsondiff.diff( + raw_data, + { + "name": "TestPackage", + "version": "1.2.3", + "description": "This is Arduino library", + "sentence": "This is Arduino library", + "frameworks": ["arduino"], + "authors": [ + {"name": "SomeAuthor", "email": "info@author.com"}, + {"name": "Maintainer Author", "maintainer": True}, + ], + "category": "Signal Input/Output", + "keywords": ["signal", "input", "output"], + "headers": ["Arduino.h", "Arduino Space.hpp"], + "includes": "Arduino.h, Arduino Space.hpp", + "customField": "Custom Value", + "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", + "dependencies": [ + { + "name": "First Library", + "version": "=2.0.0", + "frameworks": ["arduino"], + }, + { + "name": "Second Library", + "version": ">=1.2.0", + "frameworks": ["arduino"], + }, + {"name": "Third", "frameworks": ["arduino"]}, + ], + }, + ) + + # Platforms ALL + data = parser.LibraryPropertiesManifestParser( + "architectures=*\n" + contents + ).as_dict() + assert data["platforms"] == ["*"] + + # Platforms specific + data = parser.LibraryPropertiesManifestParser( + "architectures=avr, esp32\n" + contents + ).as_dict() + assert data["platforms"] == ["atmelavr", "espressif32"] + + # Remote URL + data = parser.LibraryPropertiesManifestParser( + contents, + remote_url=( + "https://raw.githubusercontent.com/username/reponame/master/" + "libraries/TestPackage/library.properties" + ), + ).as_dict() + assert data["export"] == { + "include": ["libraries/TestPackage"], + } + assert data["repository"] == { + "url": "https://github.com/username/reponame.git", + "type": "git", + } + + # Home page + data = parser.LibraryPropertiesManifestParser( + "url=https://github.com/username/reponame.git\n" + contents + ).as_dict() + assert data["repository"] == { + "type": "git", + "url": "https://github.com/username/reponame.git", + } + + # Author + Maintainer + data = parser.LibraryPropertiesManifestParser( + """ +author=Rocket Scream Electronics +maintainer=Rocket Scream Electronics +""" + ).as_dict() + assert data["authors"] == [ + {"name": "Rocket Scream Electronics", "maintainer": True} + ] + assert "keywords" not in data + + +def test_library_json_schema(): + contents = """ +{ + "name": "ArduinoJson", + "keywords": "JSON, rest, http, web", + "description": "An elegant and efficient JSON library for embedded systems", + "homepage": "https://arduinojson.org", + "repository": { + "type": "git", + "url": "https://github.com/bblanchon/ArduinoJson.git" + }, + "version": "6.12.0", + "authors": { + "name": "Benoit Blanchon", + "url": "https://blog.benoitblanchon.fr" + }, + "downloadUrl": "https://example.com/package.tar.gz", + "exclude": [ + "fuzzing", + "scripts", + "test", + "third-party" + ], + "frameworks": "arduino", + "platforms": "*", + "license": "MIT", + "scripts": { + "postinstall": "script.py" + }, + "examples": [ + { + "name": "JsonConfigFile", + "base": "examples/JsonConfigFile", + "files": ["JsonConfigFile.ino"] + }, + { + "name": "JsonHttpClient", + "base": "examples/JsonHttpClient", + "files": ["JsonHttpClient.ino"] + } + ], + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino"}, + {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} + ] +} +""" + raw_data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.LIBRARY_JSON + ).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + + data = ManifestSchema().load_manifest(raw_data) + + assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git" + assert data["examples"][1]["base"] == "examples/JsonHttpClient" + assert data["examples"][1]["files"] == ["JsonHttpClient.ino"] + + assert not jsondiff.diff( + data, + { + "name": "ArduinoJson", + "keywords": ["json", "rest", "http", "web"], + "description": "An elegant and efficient JSON library for embedded systems", + "homepage": "https://arduinojson.org", + "repository": { + "url": "https://github.com/bblanchon/ArduinoJson.git", + "type": "git", + }, + "version": "6.12.0", + "authors": [ + {"name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr"} + ], + "downloadUrl": "https://example.com/package.tar.gz", + "export": {"exclude": ["fuzzing", "scripts", "test", "third-party"]}, + "frameworks": ["arduino"], + "platforms": ["*"], + "license": "MIT", + "scripts": {"postinstall": "script.py"}, + "examples": [ + { + "name": "JsonConfigFile", + "base": "examples/JsonConfigFile", + "files": ["JsonConfigFile.ino"], + }, + { + "name": "JsonHttpClient", + "base": "examples/JsonHttpClient", + "files": ["JsonHttpClient.ino"], + }, + ], + "dependencies": [ + {"name": "@owner/deps2", "version": "1.0.0", "frameworks": ["arduino"]}, + {"name": "deps1", "version": "1.0.0"}, + { + "name": "deps3", + "version": "1.0.0", + "platforms": ["ststm32", "sifive"], + }, + ], + }, + ) + + # legacy dependencies format + contents = """ +{ + "name": "DallasTemperature", + "version": "3.8.0", + "dependencies": + { + "name": "OneWire", + "authors": "Paul Stoffregen", + "frameworks": "arduino" + } +} +""" + raw_data = parser.LibraryJsonManifestParser(contents).as_dict() + data = ManifestSchema().load_manifest(raw_data) + assert not jsondiff.diff( + data, + { + "name": "DallasTemperature", + "version": "3.8.0", + "dependencies": [ + { + "name": "OneWire", + "authors": ["Paul Stoffregen"], + "frameworks": ["arduino"], + } + ], + }, + ) + + # test multiple licenses + contents = """ +{ + "name": "MultiLicense", + "version": "1.0.0", + "license": "MIT AND (LGPL-2.1-or-later OR BSD-3-Clause)" +} +""" + raw_data = parser.LibraryJsonManifestParser(contents).as_dict() + data = ManifestSchema().load_manifest(raw_data) + assert not jsondiff.diff( + data, + { + "name": "MultiLicense", + "version": "1.0.0", + "license": "MIT AND (LGPL-2.1-or-later OR BSD-3-Clause)", + }, + ) + + +def test_library_properties_schema(): + contents = """ +name=U8glib +version=1.19.1 +author=oliver +maintainer=oliver +sentence=A library for monochrome TFTs and OLEDs +paragraph=Supported display controller: SSD1306, SSD1309, SSD1322, SSD1325 +category=Display +url=https://github.com/olikraus/u8glib +architectures=avr,sam,samd +depends=First Library (=2.0.0), Second Library (>=1.2.0), Third +""" + raw_data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.LIBRARY_PROPERTIES + ).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + + data = ManifestSchema().load_manifest(raw_data) + + assert not jsondiff.diff( + data, + { + "description": ( + "A library for monochrome TFTs and OLEDs. Supported display " + "controller: SSD1306, SSD1309, SSD1322, SSD1325" + ), + "repository": { + "url": "https://github.com/olikraus/u8glib.git", + "type": "git", + }, + "frameworks": ["arduino"], + "platforms": ["atmelavr", "atmelsam"], + "version": "1.19.1", + "authors": [ + {"maintainer": True, "email": "olikraus@gmail.com", "name": "oliver"} + ], + "keywords": ["display"], + "name": "U8glib", + "dependencies": [ + { + "name": "First Library", + "version": "=2.0.0", + "frameworks": ["arduino"], + }, + { + "name": "Second Library", + "version": ">=1.2.0", + "frameworks": ["arduino"], + }, + {"name": "Third", "frameworks": ["arduino"]}, + ], + }, + ) + + # Broken fields + contents = """ +name=Mozzi +version=1.0.3 +author=Lorem Ipsum is simply dummy text of the printing and typesetting industry Lorem Ipsum has been the industry's standard dummy text ever since the 1500s when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries but also the leap into electronic typesetting remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. +maintainer=Tim Barrass +sentence=Sound synthesis library for Arduino +paragraph=With Mozzi, you can construct sounds using familiar synthesis units like oscillators, delays, filters and envelopes. +category=Signal Input/Output +url=https://sensorium.github.io/Mozzi/ +architectures=* +dot_a_linkage=false +includes=MozziGuts.h +""" + raw_data = parser.ManifestParserFactory.new( + contents, + parser.ManifestFileType.LIBRARY_PROPERTIES, + remote_url=( + "https://raw.githubusercontent.com/sensorium/Mozzi/" + "master/library.properties" + ), + ).as_dict() + + errors = None + try: + ManifestSchema().load_manifest(raw_data) + except ManifestValidationError as exc: + data = exc.valid_data + errors = exc.messages + + assert errors["authors"] + + assert not jsondiff.diff( + data, + { + "name": "Mozzi", + "version": "1.0.3", + "description": ( + "Sound synthesis library for Arduino. With Mozzi, you can construct " + "sounds using familiar synthesis units like oscillators, delays, " + "filters and envelopes." + ), + "repository": { + "url": "https://github.com/sensorium/Mozzi.git", + "type": "git", + }, + "platforms": ["*"], + "frameworks": ["arduino"], + "headers": ["MozziGuts.h"], + "authors": [ + { + "maintainer": True, + "email": "faveflave@gmail.com", + "name": "Tim Barrass", + } + ], + "keywords": ["signal", "input", "output"], + "homepage": "https://sensorium.github.io/Mozzi/", + }, + ) + + +def test_platform_json_schema(): + contents = """ +{ + "name": "atmelavr", + "title": "Atmel AVR", + "description": "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of performance, power efficiency and design flexibility. Optimized to speed time to market-and easily adapt to new ones-they are based on the industrys most code-efficient architecture for C and assembly programming.", + "keywords": "arduino, atmel, avr, MCU", + "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", + "license": "Apache-2.0", + "engines": { + "platformio": "<5" + }, + "repository": { + "type": "git", + "url": "https://github.com/platformio/platform-atmelavr.git" + }, + "version": "1.15.0", + "frameworks": { + "arduino": { + "package": "framework-arduinoavr", + "script": "builder/frameworks/arduino.py" + }, + "simba": { + "package": "framework-simba", + "script": "builder/frameworks/simba.py" + } + }, + "packages": { + "toolchain-atmelavr": { + "type": "toolchain", + "owner": "platformio", + "version": "~1.50400.0" + }, + "framework-arduinoavr": { + "type": "framework", + "optional": true, + "version": "~4.2.0" + }, + "tool-avrdude": { + "type": "uploader", + "optional": true, + "version": "~1.60300.0" + } + } +} +""" + raw_data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.PLATFORM_JSON + ).as_dict() + raw_data["frameworks"] = sorted(raw_data["frameworks"]) + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + + data = ManifestSchema().load_manifest(raw_data) + + assert not jsondiff.diff( + data, + { + "name": "atmelavr", + "title": "Atmel AVR", + "description": ( + "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of " + "performance, power efficiency and design flexibility. Optimized to " + "speed time to market-and easily adapt to new ones-they are based " + "on the industrys most code-efficient architecture for C and " + "assembly programming." + ), + "keywords": ["arduino", "atmel", "avr", "mcu"], + "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", + "license": "Apache-2.0", + "repository": { + "url": "https://github.com/platformio/platform-atmelavr.git", + "type": "git", + }, + "frameworks": sorted(["arduino", "simba"]), + "version": "1.15.0", + "dependencies": [ + {"name": "framework-arduinoavr", "version": "~4.2.0"}, + {"name": "tool-avrdude", "version": "~1.60300.0"}, + { + "name": "toolchain-atmelavr", + "owner": "platformio", + "version": "~1.50400.0", + }, + ], + }, + ) + + +def test_package_json_schema(): + contents = """ +{ + "name": "tool-scons", + "description": "SCons software construction tool", + "keywords": "SCons, build", + "homepage": "http://www.scons.org", + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l", "LINUX_ARMV7L"], + "version": "3.30101.0" +} +""" + raw_data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.PACKAGE_JSON + ).as_dict() + + data = ManifestSchema().load_manifest(raw_data) + + assert not jsondiff.diff( + data, + { + "name": "tool-scons", + "description": "SCons software construction tool", + "keywords": ["scons", "build"], + "homepage": "http://www.scons.org", + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], + "version": "3.30101.0", + }, + ) + + mp = parser.ManifestParserFactory.new( + '{"system": "*"}', parser.ManifestFileType.PACKAGE_JSON + ) + assert "system" not in mp.as_dict() + + mp = parser.ManifestParserFactory.new( + '{"system": "all"}', parser.ManifestFileType.PACKAGE_JSON + ) + assert "system" not in mp.as_dict() + + mp = parser.ManifestParserFactory.new( + '{"system": "darwin_x86_64"}', parser.ManifestFileType.PACKAGE_JSON + ) + assert mp.as_dict()["system"] == ["darwin_x86_64"] + + # shortcut repository syntax (npm-style) + contents = """ +{ + "name": "tool-github", + "version": "1.2.0", + "repository": "github:user/repo" +} +""" + raw_data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.PACKAGE_JSON + ).as_dict() + data = ManifestSchema().load_manifest(raw_data) + assert data["repository"]["url"] == "https://github.com/user/repo.git" + + +def test_parser_from_dir(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("package.json").write('{"name": "package.json"}') + pkg_dir.join("library.json").write('{"name": "library.json"}') + pkg_dir.join("library.properties").write("name=library.properties") + + data = parser.ManifestParserFactory.new_from_dir(str(pkg_dir)).as_dict() + assert data["name"] == "library.json" + + data = parser.ManifestParserFactory.new_from_dir( + str(pkg_dir), remote_url="http://localhost/library.properties" + ).as_dict() + assert data["name"] == "library.properties" + + +def test_examples_from_dir(tmpdir_factory): + package_dir = tmpdir_factory.mktemp("project") + package_dir.join("library.json").write( + '{"name": "pkg", "version": "1.0.0", "examples": ["examples/*/*.pde"]}' + ) + examples_dir = package_dir.mkdir("examples") + + # PlatformIO project #1 + pio_dir = examples_dir.mkdir("PlatformIO").mkdir("hello") + pio_dir.join(".vimrc").write("") + pio_ini = pio_dir.join("platformio.ini") + pio_ini.write("") + if not IS_WINDOWS: + pio_dir.join("platformio.ini.copy").mksymlinkto(pio_ini) + pio_dir.mkdir("include").join("main.h").write("") + pio_dir.mkdir("src").join("main.cpp").write("") + + # wiring examples + arduino_dir = examples_dir.mkdir("1. General") + arduino_dir.mkdir("SomeSketchIno").join("SomeSketchIno.ino").write("") + arduino_dir.mkdir("SomeSketchPde").join("SomeSketchPde.pde").write("") + + # custom examples + demo_dir = examples_dir.mkdir("demo") + demo_dir.join("demo.cpp").write("") + demo_dir.join("demo.h").write("") + demo_dir.join("util.h").write("") + + # PlatformIO project #2 + pio_dir = examples_dir.mkdir("world") + pio_dir.join("platformio.ini").write("") + pio_dir.join("README").write("") + pio_dir.join("extra.py").write("") + pio_dir.mkdir("include").join("world.h").write("") + pio_dir.mkdir("src").join("world.c").write("") + + # example files in root + examples_dir.join("root.c").write("") + examples_dir.join("root.h").write("") + + # invalid example + examples_dir.mkdir("invalid-example").join("hello.json") + + # Do testing + + raw_data = parser.ManifestParserFactory.new_from_dir(str(package_dir)).as_dict() + assert isinstance(raw_data["examples"], list) + assert len(raw_data["examples"]) == 6 + + def _to_unix_path(path): + return re.sub(r"[\\/]+", "/", path) + + def _sort_examples(items): + for i, _ in enumerate(items): + items[i]["base"] = _to_unix_path(items[i]["base"]) + items[i]["files"] = [_to_unix_path(f) for f in sorted(items[i]["files"])] + return sorted(items, key=lambda item: item["name"]) + + raw_data["examples"] = _sort_examples(raw_data["examples"]) + + data = ManifestSchema().load_manifest(raw_data) + + assert not jsondiff.diff( + data, + { + "version": "1.0.0", + "name": "pkg", + "examples": _sort_examples( + [ + { + "name": "PlatformIO/hello", + "base": os.path.join("examples", "PlatformIO", "hello"), + "files": [ + "platformio.ini", + os.path.join("include", "main.h"), + os.path.join("src", "main.cpp"), + ], + }, + { + "name": "1_General/SomeSketchIno", + "base": os.path.join("examples", "1. General", "SomeSketchIno"), + "files": ["SomeSketchIno.ino"], + }, + { + "name": "1_General/SomeSketchPde", + "base": os.path.join("examples", "1. General", "SomeSketchPde"), + "files": ["SomeSketchPde.pde"], + }, + { + "name": "demo", + "base": os.path.join("examples", "demo"), + "files": ["demo.h", "util.h", "demo.cpp"], + }, + { + "name": "world", + "base": "examples/world", + "files": [ + "platformio.ini", + os.path.join("include", "world.h"), + os.path.join("src", "world.c"), + "README", + "extra.py", + ], + }, + { + "name": "Examples", + "base": "examples", + "files": ["root.c", "root.h"], + }, + ] + ), + }, + ) + + +def test_parser_from_archive(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("package.json").write('{"name": "package.json"}') + pkg_dir.join("library.json").write('{"name": "library.json"}') + pkg_dir.join("library.properties").write("name=library.properties") + + archive_path = os.path.join(str(pkg_dir), "package.tar.gz") + with tarfile.open(archive_path, mode="w|gz") as tf: + for item in os.listdir(str(pkg_dir)): + tf.add(os.path.join(str(pkg_dir), item), item) + + data = parser.ManifestParserFactory.new_from_archive(archive_path).as_dict() + assert data["name"] == "library.json" + + +def test_broken_schemas(): + # missing required field + with pytest.raises( + ManifestValidationError, match=("Invalid semantic versioning format") + ) as exc_info: + ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version")) + assert exc_info.value.valid_data == {"name": "MyPackage"} + + # invalid StrictList + with pytest.raises( + ManifestValidationError, match=("Invalid manifest fields.+keywords") + ) as exc_info: + ManifestSchema().load_manifest( + dict(name="MyPackage", version="1.0.0", keywords=["kw1", "*^[]"]) + ) + assert list(exc_info.value.messages.keys()) == ["keywords"] + assert exc_info.value.valid_data["keywords"] == ["kw1"] + + # broken SemVer + with pytest.raises( + ManifestValidationError, match=("Invalid semantic versioning format") + ): + ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version")) + # version with leading zeros + with pytest.raises( + ManifestValidationError, match=("Invalid semantic versioning format") + ): + ManifestSchema().load_manifest(dict(name="MyPackage", version="01.02.00")) + + # broken value for Nested + with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"): + ManifestSchema().load_manifest( + dict( + name="MyPackage", + description="MyDescription", + keywords=["a", "b"], + authors=["should be dict here"], + version="1.2.3", + ) + ) + + # invalid package name + with pytest.raises(ManifestValidationError, match="are not allowed"): + ManifestSchema().load_manifest(dict(name="C/C++ :library", version="1.2.3")) diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py new file mode 100644 index 0000000..fc27453 --- /dev/null +++ b/tests/package/test_meta.py @@ -0,0 +1,337 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import jsondiff +import semantic_version + +from platformio.package.meta import ( + PackageCompatibility, + PackageMetadata, + PackageOutdatedResult, + PackageSpec, + PackageType, +) + + +def test_outdated_result(): + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0") + assert result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.5.4") + assert result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.2.3") + assert not result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", detached=True) + assert not result.is_outdated() + assert not result.is_outdated(allow_incompatible=True) + + +def test_spec_owner(): + assert PackageSpec("alice/foo library") == PackageSpec( + owner="alice", name="foo library" + ) + spec = PackageSpec(" Bob / BarUpper ") + assert spec != PackageSpec(owner="BOB", name="BARUPPER") + assert spec.owner == "Bob" + assert spec.name == "BarUpper" + + +def test_spec_id(): + assert PackageSpec(13) == PackageSpec(id=13) + assert PackageSpec("20") == PackageSpec(id=20) + spec = PackageSpec("id=199") + assert spec == PackageSpec(id=199) + assert isinstance(spec.id, int) + + +def test_spec_name(): + assert PackageSpec("foo") == PackageSpec(name="foo") + assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") + + +def test_spec_requirements(): + assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec( + name="foo", requirements=semantic_version.Version("1.2.3") + ) == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") + assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") + assert PackageSpec( + name="hello", requirements=semantic_version.SimpleSpec("~1.2.3") + ) == PackageSpec(name="hello", requirements="~1.2.3") + spec = PackageSpec("id=20 @ !=1.2.3,<2.0") + assert not spec.external + assert isinstance(spec.requirements, semantic_version.SimpleSpec) + assert semantic_version.Version("1.3.0-beta.1") in spec.requirements + assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") + + +def test_spec_local_urls(tmpdir_factory): + assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( + uri="file:///tmp/foo.tar.gz", name="foo" + ) + assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( + uri="file:///tmp/bar.zip", name="customName" + ) + assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( + uri="file:///tmp/some-lib/", name="some-lib" + ) + assert PackageSpec("symlink:///tmp/soft-link/") == PackageSpec( + uri="symlink:///tmp/soft-link/", name="soft-link" + ) + # detached package + assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec( + uri="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib" + ) + # detached folder without scheme + pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir() + assert PackageSpec(str(pkg_dir)) == PackageSpec( + name="detached", uri="file://%s" % pkg_dir + ) + + +def test_spec_external_urls(): + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip" + ) == PackageSpec( + uri="https://github.com/platformio/platformio-core/archive/develop.zip", + name="platformio-core", + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ) == PackageSpec( + uri="https://github.com/platformio/platformio-core/archive/" + "develop.zip?param=value", + name="platformio-core", + requirements="!=2", + ) + spec = PackageSpec( + "Custom-Name=" + "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" + ) + assert spec.external + assert spec.has_custom_name() + assert spec.name == "Custom-Name" + assert spec == PackageSpec( + uri="https://github.com/platformio/platformio-core/archive/develop.tar.gz", + name="Custom-Name", + requirements="4.4.0", + ) + + +def test_spec_vcs_urls(): + assert PackageSpec("https://github.com/platformio/platformio-core") == PackageSpec( + name="platformio-core", uri="git+https://github.com/platformio/platformio-core" + ) + assert PackageSpec("https://gitlab.com/username/reponame") == PackageSpec( + name="reponame", uri="git+https://gitlab.com/username/reponame" + ) + assert PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) == PackageSpec( + name="wolfSSL", uri="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core.git#master" + ) == PackageSpec( + name="platformio-core", + uri="git+https://github.com/platformio/platformio-core.git#master", + ) + assert PackageSpec( + "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" + ) == PackageSpec( + name="core", + uri="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", + requirements="4.4.0", + ) + assert PackageSpec( + "username@github.com:platformio/platformio-core.git" + ) == PackageSpec( + name="platformio-core", + uri="git+username@github.com:platformio/platformio-core.git", + ) + assert PackageSpec( + "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" + ) == PackageSpec( + name="pkg", + uri="git+git@github.com:platformio/platformio-core.git", + requirements="^1.2.3,!=5", + ) + assert PackageSpec( + owner="platformio", + name="external-repo", + requirements="https://github.com/platformio/platformio-core", + ) == PackageSpec( + owner="platformio", + name="external-repo", + uri="git+https://github.com/platformio/platformio-core", + ) + + +def test_spec_as_dict(): + assert not jsondiff.diff( + PackageSpec("bob/foo@1.2.3").as_dict(), + { + "owner": "bob", + "id": None, + "name": "foo", + "requirements": "1.2.3", + "uri": None, + }, + ) + assert not jsondiff.diff( + PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ).as_dict(), + { + "owner": None, + "id": None, + "name": "platformio-core", + "requirements": "!=2", + "uri": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value", + }, + ) + + +def test_spec_as_dependency(): + assert PackageSpec("owner/pkgname").as_dependency() == "owner/pkgname" + assert PackageSpec(owner="owner", name="pkgname").as_dependency() == "owner/pkgname" + assert PackageSpec("bob/foo @ ^1.2.3").as_dependency() == "bob/foo@^1.2.3" + assert ( + PackageSpec( + "https://github.com/o/r/a/develop.zip?param=value @ !=2" + ).as_dependency() + == "https://github.com/o/r/a/develop.zip?param=value @ !=2" + ) + assert ( + PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ).as_dependency() + == "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) + + +def test_metadata_as_dict(): + metadata = PackageMetadata(PackageType.LIBRARY, "foo", "1.2.3") + # test setter + metadata.version = "0.1.2+12345" + assert metadata.version == semantic_version.Version("0.1.2+12345") + assert not jsondiff.diff( + metadata.as_dict(), + { + "type": PackageType.LIBRARY, + "name": "foo", + "version": "0.1.2+12345", + "spec": None, + }, + ) + + assert not jsondiff.diff( + PackageMetadata( + PackageType.TOOL, + "toolchain", + "2.0.5", + PackageSpec("platformio/toolchain@~2.0.0"), + ).as_dict(), + { + "type": PackageType.TOOL, + "name": "toolchain", + "version": "2.0.5", + "spec": { + "owner": "platformio", + "id": None, + "name": "toolchain", + "requirements": "~2.0.0", + "uri": None, + }, + }, + ) + + +def test_metadata_dump(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + metadata = PackageMetadata( + PackageType.TOOL, + "toolchain", + "2.0.5", + PackageSpec("platformio/toolchain@~2.0.0"), + ) + + dst = pkg_dir.join(".piopm") + metadata.dump(str(dst)) + assert os.path.isfile(str(dst)) + contents = dst.read() + assert all(s in contents for s in ("null", '"~2.0.0"')) + + +def test_metadata_load(tmpdir_factory): + contents = """ +{ + "name": "foo", + "spec": { + "name": "foo", + "owner": "username", + "requirements": "!=3.4.5" + }, + "type": "platform", + "version": "0.1.3" +} +""" + pkg_dir = tmpdir_factory.mktemp("package") + dst = pkg_dir.join(".piopm") + dst.write(contents) + metadata = PackageMetadata.load(str(dst)) + assert metadata.version == semantic_version.Version("0.1.3") + assert metadata == PackageMetadata( + PackageType.PLATFORM, + "foo", + "0.1.3", + spec=PackageSpec(owner="username", name="foo", requirements="!=3.4.5"), + ) + + piopm_path = pkg_dir.join(".piopm") + metadata = PackageMetadata( + PackageType.LIBRARY, "mylib", version="1.2.3", spec=PackageSpec("mylib") + ) + metadata.dump(str(piopm_path)) + restored_metadata = PackageMetadata.load(str(piopm_path)) + assert metadata == restored_metadata + + +def test_compatibility(): + assert PackageCompatibility().is_compatible(PackageCompatibility()) + assert PackageCompatibility().is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility(frameworks=["arduino"]).is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility(platforms="espressif32").is_compatible( + PackageCompatibility(platforms=["espressif32"]) + ) + assert PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=None)) + assert PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=["*"])) + assert not PackageCompatibility( + platforms="espressif32", frameworks=["arduino"] + ).is_compatible(PackageCompatibility(platforms=["atmelavr"])) diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py new file mode 100644 index 0000000..558591b --- /dev/null +++ b/tests/package/test_pack.py @@ -0,0 +1,188 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import tarfile + +import pytest + +from platformio import fs +from platformio.compat import IS_WINDOWS +from platformio.package.exception import UnknownManifestError +from platformio.package.pack import PackagePacker + + +def test_base(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join(".git").mkdir().join("file").write("") + pkg_dir.join(".gitignore").write("") + pkg_dir.join("._hidden_file").write("") + pkg_dir.join("main.cpp").write("#include ") + p = PackagePacker(str(pkg_dir)) + # test missed manifest + with pytest.raises(UnknownManifestError): + p.pack() + # minimal package + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + with fs.cd(str(pkg_dir)): + p.pack() + with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set( + [".gitignore", "include/main.h", "library.json", "main.cpp"] + ) + + +def test_filters(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + src_dir = pkg_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + src_dir.mkdir("util").join("helpers.cpp").write("void") + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + test_dir = pkg_dir.mkdir("tests") + test_dir.join("test_1.h").write("") + test_dir.join("test_2.h").write("") + + # test include with remap of root + pkg_dir.join("library.json").write( + json.dumps(dict(name="bar", version="1.2.3", export={"include": "src"})) + ) + p = PackagePacker(str(pkg_dir)) + with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["util/helpers.cpp", "main.cpp", "library.json"] + ) + os.unlink(str(src_dir.join("library.json"))) + + # test include "src" and "include" + pkg_dir.join("library.json").write( + json.dumps( + dict(name="bar", version="1.2.3", export={"include": ["src", "include"]}) + ) + ) + p = PackagePacker(str(pkg_dir)) + with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["include/main.h", "library.json", "src/main.cpp", "src/util/helpers.cpp"] + ) + + # test include & exclude + pkg_dir.join("library.json").write( + json.dumps( + dict( + name="bar", + version="1.2.3", + export={"include": ["src", "include"], "exclude": ["*/*.h"]}, + ) + ) + ) + p = PackagePacker(str(pkg_dir)) + with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["library.json", "src/main.cpp", "src/util/helpers.cpp"] + ) + + +def test_gitgnore_filters(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join(".git").mkdir().join("file").write("") + pkg_dir.join(".gitignore").write( + """ +# comment + +gi_file +gi_folder +gi_folder_* + +**/main_nested.h + +gi_keep_file +!gi_keep_file +LICENSE +""" + ) + pkg_dir.join("LICENSE").write("") + pkg_dir.join("gi_keep_file").write("") + pkg_dir.join("gi_file").write("") + pkg_dir.mkdir("gi_folder").join("main.h").write("#ifndef") + pkg_dir.mkdir("gi_folder_name").join("main.h").write("#ifndef") + pkg_dir.mkdir("gi_nested_folder").mkdir("a").mkdir("b").join("main_nested.h").write( + "#ifndef" + ) + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + p = PackagePacker(str(pkg_dir)) + with fs.cd(str(pkg_dir)): + p.pack() + with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["library.json", "LICENSE", ".gitignore", "gi_keep_file"] + ) + + +def test_symlinks(tmpdir_factory): + # Windows does not support symbolic links + if IS_WINDOWS: + return + pkg_dir = tmpdir_factory.mktemp("package") + src_dir = pkg_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + src_dir.join("main.h").mksymlinkto(os.path.join("..", "include", "main.h")) + pkg_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') + tarball = pkg_dir.join("bar.tar.gz") + with tarfile.open(str(tarball), "w:gz") as tar: + for item in pkg_dir.listdir(): + tar.add(str(item), str(item.relto(pkg_dir))) + + p = PackagePacker(str(tarball)) + assert p.pack(str(pkg_dir)).endswith("bar-2.0.0.tar.gz") + with tarfile.open(os.path.join(str(pkg_dir), "bar-2.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["include/main.h", "library.json", "src/main.cpp", "src/main.h"] + ) + m = tar.getmember("src/main.h") + assert m.issym() + + +def test_source_root(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + root_dir = pkg_dir.mkdir("root") + src_dir = root_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + root_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') + p = PackagePacker(str(pkg_dir)) + with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: + assert set(tar.getnames()) == set(["library.json", "src/main.cpp"]) + + +def test_manifest_uri(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + root_dir = pkg_dir.mkdir("root") + src_dir = root_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + root_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + bar_dir = root_dir.mkdir("library").mkdir("bar") + bar_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') + bar_dir.mkdir("include").join("bar.h").write("") + + manifest_path = pkg_dir.join("remote_library.json") + manifest_path.write( + '{"name": "bar", "version": "3.0.0", "export": {"include": "root/library/bar"}}' + ) + + p = PackagePacker(str(pkg_dir), manifest_uri="file:%s" % manifest_path) + p.pack(str(pkg_dir)) + with tarfile.open(os.path.join(str(pkg_dir), "bar-2.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set(["library.json", "include/bar.h"]) diff --git a/tests/project/__init__.py b/tests/project/__init__.py new file mode 100644 index 0000000..b051490 --- /dev/null +++ b/tests/project/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/project/test_config.py b/tests/project/test_config.py new file mode 100644 index 0000000..6b74285 --- /dev/null +++ b/tests/project/test_config.py @@ -0,0 +1,717 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-outer-name + +import configparser +import os +import sys +from pathlib import Path + +import pytest + +from platformio import fs +from platformio.project.config import ProjectConfig +from platformio.project.exception import ( + InvalidEnvNameError, + InvalidProjectConfError, + UnknownEnvNamesError, +) + +BASE_CONFIG = """ +[platformio] +env_default = base, extra_2 +src_dir = ${custom.src_dir} +build_dir = ${custom.build_dir} +extra_configs = + extra_envs.ini + extra_debug.ini + +# global options per [env:*] +[env] +monitor_speed = 9600 ; inline comment +custom_monitor_speed = 115200 +lib_deps = + Lib1 ; inline comment in multi-line value + Lib2 +lib_ignore = ${custom.lib_ignore} +custom_builtin_option = ${env.build_type} + +[strict_ldf] +lib_ldf_mode = chain+ +lib_compat_mode = strict + +[monitor_custom] +monitor_speed = ${env.custom_monitor_speed} + +[strict_settings] +extends = strict_ldf, monitor_custom +build_flags = -D RELEASE + +[custom] +src_dir = source +build_dir = ~/tmp/pio-$PROJECT_HASH +debug_flags = -D RELEASE +lib_flags = -lc -lm +extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS} +lib_ignore = LibIgnoreCustom + +[env:base] +build_flags = ${custom.debug_flags} ${custom.extra_flags} +lib_compat_mode = ${strict_ldf.lib_compat_mode} +targets = + +[env:test_extends] +extends = strict_settings + +[env:inject_base_env] +debug_build_flags = + ${env.debug_build_flags} + -D CUSTOM_DEBUG_FLAG + +""" + +EXTRA_ENVS_CONFIG = """ +[env:extra_1] +build_flags = + -fdata-sections + -Wl,--gc-sections + ${custom.lib_flags} + ${custom.debug_flags} + -D SERIAL_BAUD_RATE=${this.monitor_speed} +lib_install = 574 + +[env:extra_2] +build_flags = ${custom.debug_flags} ${custom.extra_flags} +lib_ignore = ${env.lib_ignore}, Lib3 +upload_port = /dev/extra_2/port +debug_server = ${custom.debug_server} +""" + +EXTRA_DEBUG_CONFIG = """ +# Override original "custom.debug_flags" +[custom] +debug_flags = -D DEBUG=1 +debug_server = + ${platformio.packages_dir}/tool-openocd/openocd + --help +src_filter = -<*> + + + + + +[env:extra_2] +build_flags = -Og +src_filter = ${custom.src_filter} + +""" + +DEFAULT_CORE_DIR = os.path.join(fs.expanduser("~"), ".platformio") + + +@pytest.fixture(scope="module") +def config(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(BASE_CONFIG) + tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) + tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) + with tmpdir.as_cwd(): + return ProjectConfig(tmpdir.join("platformio.ini").strpath) + + +def test_empty_config(): + config = ProjectConfig("/non/existing/platformio.ini") + # unknown section + with pytest.raises(InvalidProjectConfError): + config.get("unknown_section", "unknown_option") + assert config.sections() == [] + assert config.get("section", "option", 13) == 13 + + +def test_warnings(config): + config.validate(["extra_2", "base"], silent=True) + assert len(config.warnings) == 3 + assert "lib_install" in config.warnings[1] + + with pytest.raises(UnknownEnvNamesError): + config.validate(["non-existing-env"]) + + +def test_defaults(config): + assert config.get("platformio", "core_dir") == os.path.join( + os.path.expanduser("~"), ".platformio" + ) + assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"] + assert config.get("env:extra_2", "lib_compat_mode") == "soft" + assert config.get("env:extra_2", "build_type") == "release" + assert config.get("env:extra_2", "build_type", None) is None + assert config.get("env:extra_2", "lib_archive", "no") is False + + config.expand_interpolations = False + with pytest.raises( + InvalidProjectConfError, match="No option 'lib_deps' in section: 'strict_ldf'" + ): + assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"] + config.expand_interpolations = True + + +def test_sections(config): + with pytest.raises(configparser.NoSectionError): + config.getraw("unknown_section", "unknown_option") + + assert config.sections() == [ + "platformio", + "env", + "strict_ldf", + "monitor_custom", + "strict_settings", + "custom", + "env:base", + "env:test_extends", + "env:inject_base_env", + "env:extra_1", + "env:extra_2", + ] + + +def test_envs(config): + assert config.envs() == [ + "base", + "test_extends", + "inject_base_env", + "extra_1", + "extra_2", + ] + assert config.default_envs() == ["base", "extra_2"] + assert config.get_default_env() == "base" + + +def test_options(config): + assert config.options(env="base") == [ + "build_flags", + "lib_compat_mode", + "targets", + "monitor_speed", + "custom_monitor_speed", + "lib_deps", + "lib_ignore", + "custom_builtin_option", + ] + assert config.options(env="test_extends") == [ + "extends", + "build_flags", + "monitor_speed", + "lib_ldf_mode", + "lib_compat_mode", + "custom_monitor_speed", + "lib_deps", + "lib_ignore", + "custom_builtin_option", + ] + + +def test_has_option(config): + assert config.has_option("env:base", "monitor_speed") + assert not config.has_option("custom", "monitor_speed") + assert config.has_option("env:extra_1", "lib_install") + assert config.has_option("env:test_extends", "lib_compat_mode") + assert config.has_option("env:extra_2", "src_filter") + + +def test_sysenv_options(config): + assert config.getraw("custom", "extra_flags") == "" + assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] + assert config.get("env:base", "upload_port") is None + assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" + os.environ["PLATFORMIO_BUILD_FLAGS"] = "-DSYSENVDEPS1 -DSYSENVDEPS2" + os.environ["PLATFORMIO_BUILD_UNFLAGS"] = "-DREMOVE_MACRO" + os.environ["PLATFORMIO_UPLOAD_PORT"] = "/dev/sysenv/port" + os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] = "-L /usr/local/lib" + assert config.get("custom", "extra_flags") == "-L /usr/local/lib" + assert config.get("env:base", "build_flags") == [ + "-D DEBUG=1 -L /usr/local/lib", + "-DSYSENVDEPS1 -DSYSENVDEPS2", + ] + assert config.get("env:base", "upload_port") == "/dev/sysenv/port" + assert config.get("env:extra_2", "upload_port") == "/dev/sysenv/port" + assert config.get("env:base", "build_unflags") == ["-DREMOVE_MACRO"] + + # env var as option + assert config.options(env="test_extends") == [ + "extends", + "build_flags", + "monitor_speed", + "lib_ldf_mode", + "lib_compat_mode", + "custom_monitor_speed", + "lib_deps", + "lib_ignore", + "custom_builtin_option", + "build_unflags", + "upload_port", + ] + + # sysenv dirs + custom_core_dir = os.path.join(os.getcwd(), "custom-core") + custom_src_dir = os.path.join(os.getcwd(), "custom-src") + custom_build_dir = os.path.join(os.getcwd(), "custom-build") + os.environ["PLATFORMIO_HOME_DIR"] = custom_core_dir + os.environ["PLATFORMIO_SRC_DIR"] = custom_src_dir + os.environ["PLATFORMIO_BUILD_DIR"] = custom_build_dir + assert os.path.realpath(config.get("platformio", "core_dir")) == os.path.realpath( + custom_core_dir + ) + assert os.path.realpath(config.get("platformio", "src_dir")) == os.path.realpath( + custom_src_dir + ) + assert os.path.realpath(config.get("platformio", "build_dir")) == os.path.realpath( + custom_build_dir + ) + + # cleanup system environment variables + del os.environ["PLATFORMIO_BUILD_FLAGS"] + del os.environ["PLATFORMIO_BUILD_UNFLAGS"] + del os.environ["PLATFORMIO_UPLOAD_PORT"] + del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] + del os.environ["PLATFORMIO_HOME_DIR"] + del os.environ["PLATFORMIO_SRC_DIR"] + del os.environ["PLATFORMIO_BUILD_DIR"] + + +def test_getraw_value(config): + # unknown option + with pytest.raises(configparser.NoOptionError): + config.getraw("custom", "unknown_option") + # unknown option even if exists in [env] + with pytest.raises(configparser.NoOptionError): + config.getraw("platformio", "monitor_speed") + + # default + assert config.getraw("unknown", "option", "default") == "default" + assert config.getraw("env:base", "custom_builtin_option") == "release" + + # known + assert config.getraw("env:base", "targets") == "" + assert config.getraw("env:extra_1", "lib_deps") == "574" + assert config.getraw("env:extra_1", "build_flags") == ( + "\n-fdata-sections\n-Wl,--gc-sections\n" + "-lc -lm\n-D DEBUG=1\n-D SERIAL_BAUD_RATE=9600" + ) + + # extended + assert config.getraw("env:test_extends", "lib_ldf_mode") == "chain+" + assert config.getraw("env", "monitor_speed") == "9600" + assert config.getraw("env:test_extends", "monitor_speed") == "115200" + + # dir options + packages_dir = os.path.join(DEFAULT_CORE_DIR, "packages") + assert config.get("platformio", "packages_dir") == packages_dir + assert ( + config.getraw("custom", "debug_server") + == f"\n{packages_dir}/tool-openocd/openocd\n--help" + ) + assert config.getraw("platformio", "build_dir") == "~/tmp/pio-$PROJECT_HASH" + + # renamed option + assert config.getraw("env:extra_1", "lib_install") == "574" + assert config.getraw("env:extra_1", "lib_deps") == "574" + assert config.getraw("env:base", "debug_load_cmd") == ["load"] + + +def test_get_value(config): + assert config.get("custom", "debug_flags") == "-D DEBUG=1" + assert config.get("env:extra_1", "build_flags") == [ + "-fdata-sections", + "-Wl,--gc-sections", + "-lc -lm", + "-D DEBUG=1", + "-D SERIAL_BAUD_RATE=9600", + ] + assert config.get("env:extra_2", "build_flags") == ["-Og"] + assert config.get("env:extra_2", "monitor_speed") == 9600 + assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] + + # get default value from ConfigOption + assert config.get("env:inject_base_env", "debug_build_flags") == [ + "-Og", + "-g2", + "-ggdb2", + "-D CUSTOM_DEBUG_FLAG", + ] + + # dir options + assert config.get("platformio", "packages_dir") == os.path.join( + DEFAULT_CORE_DIR, "packages" + ) + assert config.get("env:extra_2", "debug_server") == [ + os.path.join(DEFAULT_CORE_DIR, "packages/tool-openocd/openocd"), + "--help", + ] + # test relative dir + assert config.get("platformio", "src_dir") == os.path.abspath( + os.path.join(os.getcwd(), "source") + ) + assert "$PROJECT_HASH" not in config.get("platformio", "build_dir") + + # renamed option + assert config.get("env:extra_1", "lib_install") == ["574"] + assert config.get("env:extra_1", "lib_deps") == ["574"] + assert config.get("env:base", "debug_load_cmd") == ["load"] + + +def test_items(config): + assert config.items("custom") == [ + ("src_dir", "source"), + ("build_dir", "~/tmp/pio-$PROJECT_HASH"), + ("debug_flags", "-D DEBUG=1"), + ("lib_flags", "-lc -lm"), + ("extra_flags", ""), + ("lib_ignore", "LibIgnoreCustom"), + ( + "debug_server", + "\n%s/tool-openocd/openocd\n--help" + % os.path.join(DEFAULT_CORE_DIR, "packages"), + ), + ("src_filter", "-<*>\n+\n+"), + ] + assert config.items(env="base") == [ + ("build_flags", ["-D DEBUG=1"]), + ("lib_compat_mode", "strict"), + ("targets", []), + ("monitor_speed", 9600), + ("custom_monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ("custom_builtin_option", "release"), + ] + assert config.items(env="extra_1") == [ + ( + "build_flags", + [ + "-fdata-sections", + "-Wl,--gc-sections", + "-lc -lm", + "-D DEBUG=1", + "-D SERIAL_BAUD_RATE=9600", + ], + ), + ("lib_install", ["574"]), + ("monitor_speed", 9600), + ("custom_monitor_speed", "115200"), + ("lib_deps", ["574"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ("custom_builtin_option", "release"), + ] + assert config.items(env="extra_2") == [ + ("build_flags", ["-Og"]), + ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), + ("upload_port", "/dev/extra_2/port"), + ( + "debug_server", + [ + "%s/tool-openocd/openocd" % os.path.join(DEFAULT_CORE_DIR, "packages"), + "--help", + ], + ), + ("src_filter", ["-<*>", "+", "+ +"]), + ("monitor_speed", 9600), + ("custom_monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("custom_builtin_option", "release"), + ] + assert config.items(env="test_extends") == [ + ("extends", ["strict_settings"]), + ("build_flags", ["-D RELEASE"]), + ("monitor_speed", 115200), + ("lib_ldf_mode", "chain+"), + ("lib_compat_mode", "strict"), + ("custom_monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ("custom_builtin_option", "release"), + ] + + +def test_update_and_save(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write( + """ +[platformio] +extra_configs = a.ini, b.ini + +[env:myenv] +board = myboard + """ + ) + config = ProjectConfig(tmpdir.join("platformio.ini").strpath) + assert config.envs() == ["myenv"] + assert config.as_tuple()[0][1][0][1] == ["a.ini", "b.ini"] + + config.update( + [ + ["platformio", [("extra_configs", ["extra.ini"])]], + ["env:myenv", [("framework", ["espidf", "arduino"])]], + ["check_types", [("float_option", 13.99), ("bool_option", True)]], + ] + ) + assert config.get("platformio", "extra_configs") == ["extra.ini"] + config.remove_section("platformio") + assert config.as_tuple() == [ + ("env:myenv", [("board", "myboard"), ("framework", ["espidf", "arduino"])]), + ("check_types", [("float_option", "13.99"), ("bool_option", "yes")]), + ] + + config.save() + contents = tmpdir.join("platformio.ini").read() + assert contents[-4:] == "yes\n" + lines = [ + line.strip() + for line in contents.split("\n") + if line.strip() and not line.startswith((";", "#")) + ] + assert lines == [ + "[env:myenv]", + "board = myboard", + "framework =", + "espidf", + "arduino", + "[check_types]", + "float_option = 13.99", + "bool_option = yes", + ] + + +def test_update_and_clear(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write( + """ +[platformio] +extra_configs = a.ini, b.ini + +[env:myenv] +board = myboard + """ + ) + config = ProjectConfig(tmpdir.join("platformio.ini").strpath) + assert config.sections() == ["platformio", "env:myenv"] + config.update([["mysection", [("opt1", "value1"), ("opt2", "value2")]]], clear=True) + assert config.as_tuple() == [ + ("mysection", [("opt1", "value1"), ("opt2", "value2")]) + ] + + +def test_dump(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(BASE_CONFIG) + tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) + tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) + config = ProjectConfig( + tmpdir.join("platformio.ini").strpath, + parse_extra=False, + expand_interpolations=False, + ) + assert config.as_tuple() == [ + ( + "platformio", + [ + ("env_default", ["base", "extra_2"]), + ("src_dir", "${custom.src_dir}"), + ("build_dir", "${custom.build_dir}"), + ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), + ], + ), + ( + "env", + [ + ("monitor_speed", 9600), + ("custom_monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["${custom.lib_ignore}"]), + ("custom_builtin_option", "${env.build_type}"), + ], + ), + ("strict_ldf", [("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict")]), + ("monitor_custom", [("monitor_speed", "${env.custom_monitor_speed}")]), + ( + "strict_settings", + [("extends", "strict_ldf, monitor_custom"), ("build_flags", "-D RELEASE")], + ), + ( + "custom", + [ + ("src_dir", "source"), + ("build_dir", "~/tmp/pio-$PROJECT_HASH"), + ("debug_flags", "-D RELEASE"), + ("lib_flags", "-lc -lm"), + ("extra_flags", "${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}"), + ("lib_ignore", "LibIgnoreCustom"), + ], + ), + ( + "env:base", + [ + ("build_flags", ["${custom.debug_flags} ${custom.extra_flags}"]), + ("lib_compat_mode", "${strict_ldf.lib_compat_mode}"), + ("targets", []), + ], + ), + ("env:test_extends", [("extends", ["strict_settings"])]), + ( + "env:inject_base_env", + [ + ( + "debug_build_flags", + ["${env.debug_build_flags}", "-D CUSTOM_DEBUG_FLAG"], + ) + ], + ), + ] + + +@pytest.mark.skipif(sys.platform != "win32", reason="runs only on windows") +def test_win_core_root_dir(tmpdir_factory): + try: + win_core_root_dir = os.path.splitdrive(fs.expanduser("~"))[0] + "\\.platformio" + remove_dir_at_exit = False + if not os.path.isdir(win_core_root_dir): + remove_dir_at_exit = True + os.makedirs(win_core_root_dir) + + # Default config + config = ProjectConfig() + assert config.get("platformio", "core_dir") == win_core_root_dir + assert config.get("platformio", "packages_dir") == os.path.join( + win_core_root_dir, "packages" + ) + + # Override in config + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write( + """ +[platformio] +core_dir = ~/.pio + """ + ) + config = ProjectConfig(tmpdir.join("platformio.ini").strpath) + assert config.get("platformio", "core_dir") != win_core_root_dir + assert config.get("platformio", "core_dir") == os.path.realpath( + fs.expanduser("~/.pio") + ) + + if remove_dir_at_exit: + fs.rmtree(win_core_root_dir) + except PermissionError: + pass + + +def test_this(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[common] +board = uno + +[env:myenv] +extends = common +build_flags = -D${this.__env__} +custom_option = ${this.board} + """ + ) + config = ProjectConfig(str(project_conf)) + assert config.get("env:myenv", "custom_option") == "uno" + assert config.get("env:myenv", "build_flags") == ["-Dmyenv"] + + +def test_nested_interpolation(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[platformio] +build_dir = ~/tmp/pio-$PROJECT_HASH + +[env:myenv] +test_testing_command = + ${platformio.packages_dir}/tool-simavr/bin/simavr + -m + atmega328p + -f + 16000000L + ${platformio.build_dir}/${this.__env__}/firmware.elf + """ + ) + config = ProjectConfig(str(project_conf)) + testing_command = config.get("env:myenv", "test_testing_command") + assert "$" not in " ".join(testing_command) + + +def test_extends_order(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[a] +board = test + +[b] +upload_tool = two + +[c] +upload_tool = three + +[env:na_ti-ve13] +extends = a, b, c + """ + ) + config = ProjectConfig(str(project_conf)) + assert config.get("env:na_ti-ve13", "upload_tool") == "three" + + +def test_invalid_env_names(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app:1] + """ + ) + config = ProjectConfig(str(project_conf)) + with pytest.raises(InvalidEnvNameError, match=r".*Invalid environment name 'app:1"): + config.validate() + + +def test_linting_errors(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app1] +lib_use = 1 +broken_line + """ + ) + result = ProjectConfig.lint(str(project_conf)) + assert not result["warnings"] + assert result["errors"] and len(result["errors"]) == 1 + error = result["errors"][0] + assert error["type"] == "ParsingError" + assert error["lineno"] == 4 + + +def test_linting_warnings(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app1] +lib_use = 1 + """ + ) + result = ProjectConfig.lint(str(project_conf)) + assert not result["errors"] + assert result["warnings"] and len(result["warnings"]) == 1 + assert "deprecated" in result["warnings"][0] diff --git a/tests/project/test_savedeps.py b/tests/project/test_savedeps.py new file mode 100644 index 0000000..2a16511 --- /dev/null +++ b/tests/project/test_savedeps.py @@ -0,0 +1,224 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.meta import PackageSpec +from platformio.project.config import ProjectConfig +from platformio.project.savedeps import save_project_dependencies + +PROJECT_CONFIG_TPL = """ +[env] +board = uno +framework = arduino +lib_deps = + SPI +platform_packages = + platformio/tool-jlink@^1.75001.0 + +[env:bare] + +[env:release] +platform = platformio/espressif32 +lib_deps = + milesburton/DallasTemperature@^3.8 + +[env:debug] +platform = platformio/espressif32@^3.4.0 +lib_deps = + ${env.lib_deps} + milesburton/DallasTemperature@^3.9.1 + bblanchon/ArduinoJson +platform_packages = + ${env.platform_packages} + platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git +""" + + +def test_save_libraries(tmp_path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + specs = [ + PackageSpec("milesburton/DallasTemperature@^3.9"), + PackageSpec("adafruit/Adafruit GPS Library@^1.6.0"), + PackageSpec("https://github.com/nanopb/nanopb.git"), + ] + + # add to the specified environment + save_project_dependencies( + str(project_dir), specs, scope="lib_deps", action="add", environments=["debug"] + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:debug", "lib_deps") == [ + "SPI", + "bblanchon/ArduinoJson", + "milesburton/DallasTemperature@^3.9", + "adafruit/Adafruit GPS Library@^1.6.0", + "https://github.com/nanopb/nanopb.git", + ] + assert config.get("env:bare", "lib_deps") == ["SPI"] + assert config.get("env:release", "lib_deps") == [ + "milesburton/DallasTemperature@^3.8" + ] + + # add to the the all environments + save_project_dependencies(str(project_dir), specs, scope="lib_deps", action="add") + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:debug", "lib_deps") == [ + "SPI", + "bblanchon/ArduinoJson", + "milesburton/DallasTemperature@^3.9", + "adafruit/Adafruit GPS Library@^1.6.0", + "https://github.com/nanopb/nanopb.git", + ] + assert config.get("env:bare", "lib_deps") == [ + "milesburton/DallasTemperature@^3.9", + "adafruit/Adafruit GPS Library@^1.6.0", + "https://github.com/nanopb/nanopb.git", + ] + assert config.get("env:release", "lib_deps") == [ + "milesburton/DallasTemperature@^3.9", + "adafruit/Adafruit GPS Library@^1.6.0", + "https://github.com/nanopb/nanopb.git", + ] + + # remove deps from env + save_project_dependencies( + str(project_dir), + [PackageSpec("milesburton/DallasTemperature")], + scope="lib_deps", + action="remove", + environments=["release"], + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:release", "lib_deps") == [ + "adafruit/Adafruit GPS Library@^1.6.0", + "https://github.com/nanopb/nanopb.git", + ] + # invalid requirements + save_project_dependencies( + str(project_dir), + [PackageSpec("adafruit/Adafruit GPS Library@^9.9.9")], + scope="lib_deps", + action="remove", + environments=["release"], + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:release", "lib_deps") == [ + "https://github.com/nanopb/nanopb.git", + ] + + # remove deps from all envs + save_project_dependencies( + str(project_dir), specs, scope="lib_deps", action="remove" + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:debug", "lib_deps") == [ + "SPI", + "bblanchon/ArduinoJson", + ] + assert config.get("env:bare", "lib_deps") == ["SPI"] + assert config.get("env:release", "lib_deps") == ["SPI"] + + +def test_save_tools(tmp_path): + project_dir = tmp_path / "project" + project_dir.mkdir() + (project_dir / "platformio.ini").write_text(PROJECT_CONFIG_TPL) + specs = [ + PackageSpec("platformio/framework-espidf@^2"), + PackageSpec("platformio/tool-esptoolpy"), + ] + + # add to the specified environment + save_project_dependencies( + str(project_dir), + specs, + scope="platform_packages", + action="add", + environments=["debug"], + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:debug", "platform_packages") == [ + "platformio/tool-jlink@^1.75001.0", + "platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git", + "platformio/framework-espidf@^2", + "platformio/tool-esptoolpy", + ] + assert config.get("env:bare", "platform_packages") == [ + "platformio/tool-jlink@^1.75001.0" + ] + assert config.get("env:release", "platform_packages") == [ + "platformio/tool-jlink@^1.75001.0" + ] + + # add to the the all environments + save_project_dependencies( + str(project_dir), specs, scope="platform_packages", action="add" + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:debug", "platform_packages") == [ + "platformio/tool-jlink@^1.75001.0", + "platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git", + "platformio/framework-espidf@^2", + "platformio/tool-esptoolpy", + ] + assert config.get("env:bare", "platform_packages") == [ + "platformio/framework-espidf@^2", + "platformio/tool-esptoolpy", + ] + assert config.get("env:release", "platform_packages") == [ + "platformio/framework-espidf@^2", + "platformio/tool-esptoolpy", + ] + + # remove deps from env + save_project_dependencies( + str(project_dir), + [PackageSpec("platformio/framework-espidf")], + scope="platform_packages", + action="remove", + environments=["release"], + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:release", "platform_packages") == [ + "platformio/tool-esptoolpy", + ] + # invalid requirements + save_project_dependencies( + str(project_dir), + [PackageSpec("platformio/tool-esptoolpy@9.9.9")], + scope="platform_packages", + action="remove", + environments=["release"], + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:release", "platform_packages") == [ + "platformio/tool-jlink@^1.75001.0", + ] + + # remove deps from all envs + save_project_dependencies( + str(project_dir), specs, scope="platform_packages", action="remove" + ) + config = ProjectConfig.get_instance(str(project_dir / "platformio.ini")) + assert config.get("env:debug", "platform_packages") == [ + "platformio/tool-jlink@^1.75001.0", + "platformio/framework-arduinoespressif32 @ https://github.com/espressif/arduino-esp32.git", + ] + assert config.get("env:bare", "platform_packages") == [ + "platformio/tool-jlink@^1.75001.0" + ] + assert config.get("env:release", "platform_packages") == [ + "platformio/tool-jlink@^1.75001.0" + ] diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..b49aa90 --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,102 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random +from glob import glob + +import pytest + +from platformio import fs, proc +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectError + + +def pytest_generate_tests(metafunc): + if "pioproject_dir" not in metafunc.fixturenames: + return + examples_dirs = [] + + # repo examples + examples_dirs.append( + os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "examples")) + ) + + # dev/platforms + for pkg in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(pkg) + examples_dir = os.path.join(p.get_dir(), "examples") + if os.path.isdir(examples_dir): + examples_dirs.append(examples_dir) + + project_dirs = [] + for examples_dir in examples_dirs: + candidates = {} + for root, _, files in os.walk(examples_dir): + if "platformio.ini" not in files or ".skiptest" in files: + continue + if "mbed-legacy-examples" in root: + continue + group = os.path.basename(root) + if "-" in group: + group = group.split("-", 1)[0] + if group not in candidates: + candidates[group] = [] + candidates[group].append(root) + + project_dirs.extend( + [random.choice(examples) for examples in candidates.values() if examples] + ) + + metafunc.parametrize("pioproject_dir", sorted(project_dirs)) + + +def test_run(pioproject_dir): + with fs.cd(pioproject_dir): + config = ProjectConfig() + + # temporary fix for unreleased dev-platforms with broken env name + try: + config.validate() + except ProjectError as exc: + pytest.skip(str(exc)) + + build_dir = config.get("platformio", "build_dir") + if os.path.isdir(build_dir): + fs.rmtree(build_dir) + + env_names = config.envs() + result = proc.exec_command( + ["platformio", "run", "-e", random.choice(env_names)] + ) + if result["returncode"] != 0: + pytest.fail(str(result)) + + assert os.path.isdir(build_dir) + + # check .elf file + for item in os.listdir(build_dir): + if not os.path.isdir(item): + continue + assert os.path.isfile(os.path.join(build_dir, item, "firmware.elf")) + # check .hex or .bin files + firmwares = [] + for ext in ("bin", "hex"): + firmwares += glob(os.path.join(build_dir, item, "firmware*.%s" % ext)) + if not firmwares: + pytest.fail("Missed firmware file") + for firmware in firmwares: + assert os.path.getsize(firmware) > 0 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..2eda9b9 --- /dev/null +++ b/tox.ini @@ -0,0 +1,71 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[isort] +profile = black +known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope + +[pytest] +filterwarnings = + error + # Bottle + ignore:.*'cgi' is deprecated and slated for removal + +[testenv] +passenv = * +usedevelop = True +deps = + black + isort + pylint + pytest + pytest-xdist + jsondiff +commands = + {envpython} --version + pio system info + +[testenv:lint] +commands = + {envpython} --version + pylint --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./tests + +[testenv:testcore] +commands = + {envpython} --version + py.test -v --basetemp={envtmpdir} -k "not skip_ci" tests --ignore tests/test_examples.py + +[testenv:testexamples] +commands = + {envpython} scripts/install_devplatforms.py + py.test -v --basetemp={envtmpdir} tests/test_examples.py + +[testenv:docs] +deps = + sphinx + sphinx-rtd-theme==1.2.2 + sphinx-notfound-page + sphinx-copybutton + restructuredtext-lint +change_dir = docs +commands = + sphinx-build -b html . _build/html + +[testenv:docslinkcheck] +deps = + {[testenv:docs]deps} +change_dir = docs +commands = + sphinx-build -b linkcheck . _build