From 95dbf2d6669d8bdeb9190c2622954e3bcf3f00db Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 8 Apr 2022 10:06:51 +0200 Subject: [PATCH 1/7] Swift: first skeleton extractor This adds a first dummy extractor for swift. Running `bazel run //swift:install` will create an `extractor_pack` directory in `swift`. From that moment providing `--search-path=swift` will pick up the extractor. --- .bazelrc | 3 ++ BUILD.bazel | 0 WORKSPACE.bazel | 43 ++++++++++++++++++++ cpp/BUILD.bazel | 6 +-- defs.bzl | 19 +++++++++ swift/.clang-format | 7 ++++ swift/.codeqlmanifest.json | 7 ++++ swift/.gitignore | 1 + swift/BUILD.bazel | 66 +++++++++++++++++++++++++++++++ swift/README.md | 8 ++++ swift/codeql-extractor.yml | 10 +++++ swift/extractor/BUILD.bazel | 5 +++ swift/extractor/main.cpp | 14 +++++++ swift/ql/lib/qlpack.yml | 5 +++ swift/ql/lib/swift.dbscheme | 7 ++++ swift/ql/lib/swift.dbscheme.stats | 4 ++ swift/ql/test/answer.expected | 1 + swift/ql/test/answer.ql | 3 ++ swift/ql/test/qlpack.yml | 5 +++ swift/tools/qltest.sh | 5 +++ 20 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 .bazelrc create mode 100644 BUILD.bazel create mode 100644 defs.bzl create mode 100644 swift/.clang-format create mode 100644 swift/.codeqlmanifest.json create mode 100644 swift/.gitignore create mode 100644 swift/BUILD.bazel create mode 100644 swift/README.md create mode 100644 swift/codeql-extractor.yml create mode 100644 swift/extractor/BUILD.bazel create mode 100644 swift/extractor/main.cpp create mode 100644 swift/ql/lib/qlpack.yml create mode 100644 swift/ql/lib/swift.dbscheme create mode 100644 swift/ql/lib/swift.dbscheme.stats create mode 100644 swift/ql/test/answer.expected create mode 100644 swift/ql/test/answer.ql create mode 100644 swift/ql/test/qlpack.yml create mode 100755 swift/tools/qltest.sh diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 000000000000..cd6226d01710 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,3 @@ +build --copt="-std=c++17" + +try-import %workspace%/local.bazelrc diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index c9357d0f4480..5fb6f875f82c 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -1,2 +1,45 @@ # Please notice that any bazel targets and definitions in this repository are currently experimental # and for internal use only. + +workspace(name = "ql") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_pkg", + sha256 = "62eeb544ff1ef41d786e329e1536c1d541bb9bcad27ae984d57f18f314018e66", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", + ], +) + +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() + +http_archive( + name = "platforms", + sha256 = "460caee0fa583b908c622913334ec3c1b842572b9c23cf0d3da0c2543a1a157d", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.3/platforms-0.0.3.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.3/platforms-0.0.3.tar.gz", + ], +) + +http_archive( + name = "bazel_skylib", + sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + ], +) + +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") + +bazel_skylib_workspace() + +load("@ql//:defs.bzl", "ql_utils") + +ql_utils(name = "utils") diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index 5341bd203908..31495c5eb1a4 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -4,14 +4,14 @@ load("@rules_pkg//:mappings.bzl", "pkg_filegroup") alias( name = "dbscheme", - actual = "//cpp/ql/lib:dbscheme", + actual = "@ql//cpp/ql/lib:dbscheme", ) pkg_filegroup( name = "db-files", srcs = [ ":dbscheme", - "//cpp/downgrades", - "//cpp/ql/lib:dbscheme-stats", + "@ql//cpp/downgrades", + "@ql//cpp/ql/lib:dbscheme-stats", ], ) diff --git a/defs.bzl b/defs.bzl new file mode 100644 index 000000000000..de0179d0cd1c --- /dev/null +++ b/defs.bzl @@ -0,0 +1,19 @@ +codeql_platform = select({ + "@platforms//os:linux": "linux64", + "@platforms//os:macos": "osx64", + "@platforms//os:windows": "win64", +}) + +_paths_bzl = """ +def source_dir(): + return '%s/' + native.package_name() +""" + +def _ql_utils_impl(repository_ctx): + root = repository_ctx.path(Label("@ql//:WORKSPACE.bazel")).realpath.dirname + repository_ctx.file("BUILD.bazel") + repository_ctx.file("paths.bzl", content = _paths_bzl % root) + +ql_utils = repository_rule( + implementation = _ql_utils_impl, +) diff --git a/swift/.clang-format b/swift/.clang-format new file mode 100644 index 000000000000..96d2febc1c2c --- /dev/null +++ b/swift/.clang-format @@ -0,0 +1,7 @@ +BasedOnStyle: Chromium +ColumnLimit: 100 +IndentWidth: 2 +SortIncludes: false +AllowShortIfStatementsOnASingleLine: WithoutElse +AlwaysBreakBeforeMultilineStrings: false +Standard: c++17 diff --git a/swift/.codeqlmanifest.json b/swift/.codeqlmanifest.json new file mode 100644 index 000000000000..4e5e2c13cb26 --- /dev/null +++ b/swift/.codeqlmanifest.json @@ -0,0 +1,7 @@ +{ + "provide": [ + "ql/lib/qlpack.yml", + "ql/test/qlpack.yml", + "extractor_pack/codeql-extractor.yml" + ] +} diff --git a/swift/.gitignore b/swift/.gitignore new file mode 100644 index 000000000000..b6256eaf7470 --- /dev/null +++ b/swift/.gitignore @@ -0,0 +1 @@ +extractor_pack diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel new file mode 100644 index 000000000000..f2a694e25e7b --- /dev/null +++ b/swift/BUILD.bazel @@ -0,0 +1,66 @@ +load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") +load("@rules_pkg//:install.bzl", "pkg_install") +load("@ql//:defs.bzl", "codeql_platform") +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") +load("@utils//:paths.bzl", "source_dir") + +pkg_files( + name = "dbscheme", + srcs = [ + "ql/lib/swift.dbscheme", + "ql/lib/swift.dbscheme.stats", + ], +) + +pkg_files( + name = "qltest", + srcs = ["tools/qltest.sh"], + attributes = pkg_attributes(mode = "0755"), + prefix = "tools", +) + +pkg_files( + name = "manifest", + srcs = ["codeql-extractor.yml"], +) + +pkg_filegroup( + name = "extractor-pack-generic", + srcs = [ + ":dbscheme", + ":manifest", + ":qltest", + ], + visibility = ["//visibility:public"], +) + +pkg_files( + name = "extractor", + srcs = ["//swift/extractor"], + attributes = pkg_attributes(mode = "0755"), + prefix = "tools/" + codeql_platform, +) + +pkg_filegroup( + name = "extractor-pack-arch", + srcs = [":extractor"], + visibility = ["//visibility:public"], +) + +pkg_filegroup( + name = "extractor-pack", + srcs = [ + ":extractor-pack-arch", + ":extractor-pack-generic", + ], + visibility = ["//visibility:public"], +) + +pkg_install( + name = "install", + srcs = [":extractor-pack"], + args = [ + "--destdir", + source_dir() + "/extractor_pack", + ], +) diff --git a/swift/README.md b/swift/README.md new file mode 100644 index 000000000000..60683e17e6ff --- /dev/null +++ b/swift/README.md @@ -0,0 +1,8 @@ +## Warning + +The Swift codeql package is an experimental and unsupported work in progress. + +## Usage + +Run `bazel run //swift:install-extractor`, which will install `swift/extractor_pack`. Using `--search-path=swift` will +then pick up the Swift extractor. diff --git a/swift/codeql-extractor.yml b/swift/codeql-extractor.yml new file mode 100644 index 000000000000..94f464fd9b58 --- /dev/null +++ b/swift/codeql-extractor.yml @@ -0,0 +1,10 @@ +name: "swift" +display_name: "Swift" +version: 0.0.1 +column_kind: "utf8" +legacy_qltest_extraction: true +file_types: + - name: swift + display_name: Swift files + extensions: + - .swift diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel new file mode 100644 index 000000000000..b201c993c43b --- /dev/null +++ b/swift/extractor/BUILD.bazel @@ -0,0 +1,5 @@ +cc_binary( + name = "extractor", + srcs = ["main.cpp"], + visibility = ["//swift:__pkg__"], +) diff --git a/swift/extractor/main.cpp b/swift/extractor/main.cpp new file mode 100644 index 000000000000..949b3fb246bf --- /dev/null +++ b/swift/extractor/main.cpp @@ -0,0 +1,14 @@ +#include +#include +#include + +int main() { + if (auto trapDir = getenv("CODEQL_EXTRACTOR_SWIFT_TRAP_DIR")) { + std::string file = trapDir; + file += "/my_first.trap"; + if (std::ofstream out{file}) { + out << "answer_to_life_the_universe_and_everything(42)\n"; + } + } + return 0; +} diff --git a/swift/ql/lib/qlpack.yml b/swift/ql/lib/qlpack.yml new file mode 100644 index 000000000000..5a6a80d2a594 --- /dev/null +++ b/swift/ql/lib/qlpack.yml @@ -0,0 +1,5 @@ +name: codeql/swift-all +version: 0.0.0 +dbscheme: swift.dbscheme +extractor: swift +library: true diff --git a/swift/ql/lib/swift.dbscheme b/swift/ql/lib/swift.dbscheme new file mode 100644 index 000000000000..3022d88cce04 --- /dev/null +++ b/swift/ql/lib/swift.dbscheme @@ -0,0 +1,7 @@ +sourceLocationPrefix( + string prefix: string ref +); + +answer_to_life_the_universe_and_everything( + int answer: int ref +) diff --git a/swift/ql/lib/swift.dbscheme.stats b/swift/ql/lib/swift.dbscheme.stats new file mode 100644 index 000000000000..9995467e33e5 --- /dev/null +++ b/swift/ql/lib/swift.dbscheme.stats @@ -0,0 +1,4 @@ + + + + diff --git a/swift/ql/test/answer.expected b/swift/ql/test/answer.expected new file mode 100644 index 000000000000..862807c972cc --- /dev/null +++ b/swift/ql/test/answer.expected @@ -0,0 +1 @@ +| 42 | diff --git a/swift/ql/test/answer.ql b/swift/ql/test/answer.ql new file mode 100644 index 000000000000..e8c6ceb7858f --- /dev/null +++ b/swift/ql/test/answer.ql @@ -0,0 +1,3 @@ +from int answer +where answer_to_life_the_universe_and_everything(answer) +select answer diff --git a/swift/ql/test/qlpack.yml b/swift/ql/test/qlpack.yml new file mode 100644 index 000000000000..1b025e1c301f --- /dev/null +++ b/swift/ql/test/qlpack.yml @@ -0,0 +1,5 @@ +name: codeql-swift-tests +version: 0.0.0 +libraryPathDependencies: + - codeql/swift-all +extractor: swift diff --git a/swift/tools/qltest.sh b/swift/tools/qltest.sh new file mode 100755 index 000000000000..1e9b098aa0e9 --- /dev/null +++ b/swift/tools/qltest.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +mkdir -p "$CODEQL_EXTRACTOR_SWIFT_TRAP_DIR" + +exec "$CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/extractor" From 664d5ba0a98741b9af4a4f539f0d49ef85754d2e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 8 Apr 2022 15:07:47 +0200 Subject: [PATCH 2/7] Swift: moved install to a separate package When importing the workspace from semmle-code, we do not need nor want to instantiate `@util`, so that must be in a separate bazel package. --- WORKSPACE.bazel | 13 ------------- swift/BUILD.bazel | 12 ------------ swift/README.md | 4 ++-- swift/install/BUILD.bazel | 11 +++++++++++ 4 files changed, 13 insertions(+), 27 deletions(-) create mode 100644 swift/install/BUILD.bazel diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 5fb6f875f82c..bc27ed44c180 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -27,19 +27,6 @@ http_archive( ], ) -http_archive( - name = "bazel_skylib", - sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d", - urls = [ - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", - ], -) - -load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") - -bazel_skylib_workspace() - load("@ql//:defs.bzl", "ql_utils") ql_utils(name = "utils") diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index f2a694e25e7b..7219f98388c1 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -1,8 +1,5 @@ load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") -load("@rules_pkg//:install.bzl", "pkg_install") load("@ql//:defs.bzl", "codeql_platform") -load("@bazel_skylib//rules:native_binary.bzl", "native_binary") -load("@utils//:paths.bzl", "source_dir") pkg_files( name = "dbscheme", @@ -55,12 +52,3 @@ pkg_filegroup( ], visibility = ["//visibility:public"], ) - -pkg_install( - name = "install", - srcs = [":extractor-pack"], - args = [ - "--destdir", - source_dir() + "/extractor_pack", - ], -) diff --git a/swift/README.md b/swift/README.md index 60683e17e6ff..e34138d0616c 100644 --- a/swift/README.md +++ b/swift/README.md @@ -4,5 +4,5 @@ The Swift codeql package is an experimental and unsupported work in progress. ## Usage -Run `bazel run //swift:install-extractor`, which will install `swift/extractor_pack`. Using `--search-path=swift` will -then pick up the Swift extractor. +Run `bazel run //swift/install`, which will install `swift/extractor_pack`. Using `--search-path=swift` will then pick +up the Swift extractor. diff --git a/swift/install/BUILD.bazel b/swift/install/BUILD.bazel new file mode 100644 index 000000000000..b270fa8f79d8 --- /dev/null +++ b/swift/install/BUILD.bazel @@ -0,0 +1,11 @@ +load("@utils//:paths.bzl", "source_dir") +load("@rules_pkg//:install.bzl", "pkg_install") + +pkg_install( + name = "install", + srcs = ["//swift:extractor-pack"], + args = [ + "--destdir", + source_dir() + "/../extractor_pack", + ], +) From 13b2442fed56eebaa482c474e0cec87a3516804c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 8 Apr 2022 15:58:33 +0200 Subject: [PATCH 3/7] Bazel: code reorganization --- WORKSPACE.bazel | 4 ++-- defs.bzl | 14 -------------- misc/bazel/BUILD.bazel | 0 misc/bazel/source_dir.bzl.tpl | 11 +++++++++++ misc/bazel/workspace.bzl | 15 +++++++++++++++ swift/install/BUILD.bazel | 4 ++-- 6 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 misc/bazel/BUILD.bazel create mode 100644 misc/bazel/source_dir.bzl.tpl create mode 100644 misc/bazel/workspace.bzl diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index bc27ed44c180..4d485742b288 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -27,6 +27,6 @@ http_archive( ], ) -load("@ql//:defs.bzl", "ql_utils") +load("@ql//misc/bazel:workspace.bzl", "ql_workspace") -ql_utils(name = "utils") +ql_workspace() diff --git a/defs.bzl b/defs.bzl index de0179d0cd1c..d6748d831761 100644 --- a/defs.bzl +++ b/defs.bzl @@ -3,17 +3,3 @@ codeql_platform = select({ "@platforms//os:macos": "osx64", "@platforms//os:windows": "win64", }) - -_paths_bzl = """ -def source_dir(): - return '%s/' + native.package_name() -""" - -def _ql_utils_impl(repository_ctx): - root = repository_ctx.path(Label("@ql//:WORKSPACE.bazel")).realpath.dirname - repository_ctx.file("BUILD.bazel") - repository_ctx.file("paths.bzl", content = _paths_bzl % root) - -ql_utils = repository_rule( - implementation = _ql_utils_impl, -) diff --git a/misc/bazel/BUILD.bazel b/misc/bazel/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/misc/bazel/source_dir.bzl.tpl b/misc/bazel/source_dir.bzl.tpl new file mode 100644 index 000000000000..245ccf6e8b8e --- /dev/null +++ b/misc/bazel/source_dir.bzl.tpl @@ -0,0 +1,11 @@ +def root_source_dir(): + """ get absolute path to the root source directory + + This can break hermeticity if used in a build step""" + return '{root}' + +def current_source_dir(): + """ get absolute path to the source directory of this bazel package + + This can break hermeticity if used in a build step""" + return root_source_dir() + '/' + native.package_name() diff --git a/misc/bazel/workspace.bzl b/misc/bazel/workspace.bzl new file mode 100644 index 000000000000..b6f9bacd2018 --- /dev/null +++ b/misc/bazel/workspace.bzl @@ -0,0 +1,15 @@ +def _ql_utils_impl(repository_ctx): + root = repository_ctx.path(Label("//:WORKSPACE.bazel")).realpath.dirname + repository_ctx.file("BUILD.bazel") + repository_ctx.template( + "source_dir.bzl", + Label("@ql//misc/bazel:source_dir.bzl.tpl"), + substitutions = {"{root}": str(root)}, + ) + +_ql_utils = repository_rule( + implementation = _ql_utils_impl, +) + +def ql_workspace(): + _ql_utils(name = "utils") diff --git a/swift/install/BUILD.bazel b/swift/install/BUILD.bazel index b270fa8f79d8..5064a265eecc 100644 --- a/swift/install/BUILD.bazel +++ b/swift/install/BUILD.bazel @@ -1,4 +1,4 @@ -load("@utils//:paths.bzl", "source_dir") +load("@utils//:source_dir.bzl", "current_source_dir") load("@rules_pkg//:install.bzl", "pkg_install") pkg_install( @@ -6,6 +6,6 @@ pkg_install( srcs = ["//swift:extractor-pack"], args = [ "--destdir", - source_dir() + "/../extractor_pack", + current_source_dir() + "/../extractor_pack", ], ) From a205b465ba4095cd5d33c7550e9e08922a272727 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 11 Apr 2022 14:29:38 +0200 Subject: [PATCH 4/7] Bazel: reorganization * fixed 5.0.0 as bazel version * made dependencies better loadable * moved `//swift/install` to `//swift:create-extractor-pack` (following the clearer ruby naming) * renamed `extractor_pack` to `extractor-pack` for consistency with Ruby --- .bazelversion | 1 + WORKSPACE.bazel | 28 ++++--------------------- misc/bazel/BUILD.toolchain.tpl | 12 +++++++++++ misc/bazel/test.bzl | 38 ++++++++++++++++++++++++++++++++++ misc/bazel/test.template.py | 3 +++ misc/bazel/workspace.bzl | 25 +++++++++++++++++++++- misc/bazel/workspace_deps.bzl | 4 ++++ swift/.codeqlmanifest.json | 2 +- swift/.gitignore | 2 +- swift/BUILD.bazel | 11 ++++++++++ swift/README.md | 5 +++-- swift/install/BUILD.bazel | 11 ---------- 12 files changed, 102 insertions(+), 40 deletions(-) create mode 100644 .bazelversion create mode 100644 misc/bazel/BUILD.toolchain.tpl create mode 100644 misc/bazel/test.bzl create mode 100644 misc/bazel/test.template.py create mode 100644 misc/bazel/workspace_deps.bzl delete mode 100644 swift/install/BUILD.bazel diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 000000000000..0062ac971805 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +5.0.0 diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 4d485742b288..b427ab37b93c 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -3,30 +3,10 @@ workspace(name = "ql") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -http_archive( - name = "rules_pkg", - sha256 = "62eeb544ff1ef41d786e329e1536c1d541bb9bcad27ae984d57f18f314018e66", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", - ], -) - -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") - -rules_pkg_dependencies() - -http_archive( - name = "platforms", - sha256 = "460caee0fa583b908c622913334ec3c1b842572b9c23cf0d3da0c2543a1a157d", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.3/platforms-0.0.3.tar.gz", - "https://github.com/bazelbuild/platforms/releases/download/0.0.3/platforms-0.0.3.tar.gz", - ], -) - load("@ql//misc/bazel:workspace.bzl", "ql_workspace") ql_workspace() + +load("@ql//misc/bazel:workspace_deps.bzl", "ql_workspace_deps") + +ql_workspace_deps() diff --git a/misc/bazel/BUILD.toolchain.tpl b/misc/bazel/BUILD.toolchain.tpl new file mode 100644 index 000000000000..849508908d86 --- /dev/null +++ b/misc/bazel/BUILD.toolchain.tpl @@ -0,0 +1,12 @@ +load("@//misc/bazel:toolchain.bzl", "codeql_cli_toolchain") + +codeql_cli_toolchain( + name = "codeql-cli", + path = "{codeql_cli_path}", +) + +toolchain( + name = "codeql-cli-toolchain", + toolchain = ":codeql-cli", + toolchain_type = "@//:toolchain_type", +) diff --git a/misc/bazel/test.bzl b/misc/bazel/test.bzl new file mode 100644 index 000000000000..24fe7dbe396b --- /dev/null +++ b/misc/bazel/test.bzl @@ -0,0 +1,38 @@ +def _test_script_impl(ctx): + output = ctx.actions.declare_file("%s.py" % ctx.label.name) + codeql_cli_path = ctx.toolchains["//:toolchain_type"].codeql_cli.path + ctx.actions.expand_template( + template = ctx.file._template, + output = output, + substitutions = { + "{codeql_cli_path}": codeql_cli_path, + "{test_sources}": str([f.path for f in ctx.files.srcs]), + }, + ) + return DefaultInfo( + files = depset([output]), + ) + +_test_script = rule( + implementation = _test_script_impl, + attrs = { + "srcs": attr.label_list(allow_files = True), + "_template": attr.label(default = "//misc/bazel:test.template.py", allow_single_file = True), + }, + toolchains = ["//:toolchain_type"], +) + +def codeql_test(*, name, srcs, deps): + srcs = native.glob(["test/**/*.ql", "test/**/*.qlref"]) + data = srcs + deps + script = name + "-script" + _test_script( + name = script, + srcs = srcs, + ) + native.py_test( + name = name, + main = script + ".py", + srcs = [script], + data = data, + ) diff --git a/misc/bazel/test.template.py b/misc/bazel/test.template.py new file mode 100644 index 000000000000..2430afb04b03 --- /dev/null +++ b/misc/bazel/test.template.py @@ -0,0 +1,3 @@ +import os + +os.execl("{codeql_cli_path}", "test", "run", "--check-databases", "--", *{test_sources}) diff --git a/misc/bazel/workspace.bzl b/misc/bazel/workspace.bzl index b6f9bacd2018..1b52c486925f 100644 --- a/misc/bazel/workspace.bzl +++ b/misc/bazel/workspace.bzl @@ -1,3 +1,6 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + def _ql_utils_impl(repository_ctx): root = repository_ctx.path(Label("//:WORKSPACE.bazel")).realpath.dirname repository_ctx.file("BUILD.bazel") @@ -12,4 +15,24 @@ _ql_utils = repository_rule( ) def ql_workspace(): - _ql_utils(name = "utils") + _ql_utils(name = "ql_utils") + + maybe( + repo_rule = http_archive, + name = "rules_pkg", + sha256 = "62eeb544ff1ef41d786e329e1536c1d541bb9bcad27ae984d57f18f314018e66", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", + ], + ) + + maybe( + repo_rule = http_archive, + name = "platforms", + sha256 = "460caee0fa583b908c622913334ec3c1b842572b9c23cf0d3da0c2543a1a157d", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.3/platforms-0.0.3.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.3/platforms-0.0.3.tar.gz", + ], + ) diff --git a/misc/bazel/workspace_deps.bzl b/misc/bazel/workspace_deps.bzl new file mode 100644 index 000000000000..a082b4b1cb7e --- /dev/null +++ b/misc/bazel/workspace_deps.bzl @@ -0,0 +1,4 @@ +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +def ql_workspace_deps(): + rules_pkg_dependencies() diff --git a/swift/.codeqlmanifest.json b/swift/.codeqlmanifest.json index 4e5e2c13cb26..3b809abfd3b0 100644 --- a/swift/.codeqlmanifest.json +++ b/swift/.codeqlmanifest.json @@ -2,6 +2,6 @@ "provide": [ "ql/lib/qlpack.yml", "ql/test/qlpack.yml", - "extractor_pack/codeql-extractor.yml" + "extractor-pack/codeql-extractor.yml" ] } diff --git a/swift/.gitignore b/swift/.gitignore index b6256eaf7470..52d331603031 100644 --- a/swift/.gitignore +++ b/swift/.gitignore @@ -1 +1 @@ -extractor_pack +extractor-pack diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 7219f98388c1..134a32d86037 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -1,5 +1,7 @@ load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") +load("@rules_pkg//:install.bzl", "pkg_install") load("@ql//:defs.bzl", "codeql_platform") +load("@ql_utils//:source_dir.bzl", "current_source_dir") pkg_files( name = "dbscheme", @@ -52,3 +54,12 @@ pkg_filegroup( ], visibility = ["//visibility:public"], ) + +pkg_install( + name = "create-extractor-pack", + srcs = ["//swift:extractor-pack"], + args = [ + "--destdir", + current_source_dir() + "/extractor-pack", + ], +) diff --git a/swift/README.md b/swift/README.md index e34138d0616c..d2b56d1a6f37 100644 --- a/swift/README.md +++ b/swift/README.md @@ -4,5 +4,6 @@ The Swift codeql package is an experimental and unsupported work in progress. ## Usage -Run `bazel run //swift/install`, which will install `swift/extractor_pack`. Using `--search-path=swift` will then pick -up the Swift extractor. +Run `bazel run //swift:create-extractor-pack`, which will install `swift/extractor-pack`. +Using `--search-path=swift/extractor-pack` will then pick up the Swift extractor. You can also use +`--search-path=swift`, as the extractor pack is mentioned in `swift/.codeqlmanifest`. diff --git a/swift/install/BUILD.bazel b/swift/install/BUILD.bazel deleted file mode 100644 index 5064a265eecc..000000000000 --- a/swift/install/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -load("@utils//:source_dir.bzl", "current_source_dir") -load("@rules_pkg//:install.bzl", "pkg_install") - -pkg_install( - name = "install", - srcs = ["//swift:extractor-pack"], - args = [ - "--destdir", - current_source_dir() + "/../extractor_pack", - ], -) From f2f99611bdd143b94e366e1e3693bc976a54ea51 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 12 Apr 2022 12:40:19 +0200 Subject: [PATCH 5/7] .gitignore CLion project files --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b87ee01c3876..33ef1770c959 100644 --- a/.gitignore +++ b/.gitignore @@ -31,5 +31,8 @@ csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json # Compiled class file *.class -# links create by bazel +# links created by bazel /bazel-* + +# CLion project files +/.clwb From 644024226884a1b25c85bfb3ebdd6d26f0291e8b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 12 Apr 2022 16:03:20 +0200 Subject: [PATCH 6/7] Swift+Bazel: apply review comments --- WORKSPACE.bazel | 4 +-- cpp/BUILD.bazel | 6 ++--- misc/bazel/BUILD.toolchain.tpl | 12 --------- misc/bazel/source_dir.bzl.tpl | 11 -------- misc/bazel/test.bzl | 38 ---------------------------- misc/bazel/test.template.py | 3 --- misc/bazel/workspace.bzl | 15 ----------- swift/BUILD.bazel | 16 +++++++----- swift/README.md | 2 +- swift/extractor/BUILD.bazel | 5 ++++ swift/ql/test/qlpack.yml | 5 ++-- swift/tools/create_extractor_pack.py | 16 ++++++++++++ 12 files changed, 39 insertions(+), 94 deletions(-) delete mode 100644 misc/bazel/BUILD.toolchain.tpl delete mode 100644 misc/bazel/source_dir.bzl.tpl delete mode 100644 misc/bazel/test.bzl delete mode 100644 misc/bazel/test.template.py create mode 100644 swift/tools/create_extractor_pack.py diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index b427ab37b93c..9459b276ff39 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -3,10 +3,10 @@ workspace(name = "ql") -load("@ql//misc/bazel:workspace.bzl", "ql_workspace") +load("//misc/bazel:workspace.bzl", "ql_workspace") ql_workspace() -load("@ql//misc/bazel:workspace_deps.bzl", "ql_workspace_deps") +load("//misc/bazel:workspace_deps.bzl", "ql_workspace_deps") ql_workspace_deps() diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index 31495c5eb1a4..5341bd203908 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -4,14 +4,14 @@ load("@rules_pkg//:mappings.bzl", "pkg_filegroup") alias( name = "dbscheme", - actual = "@ql//cpp/ql/lib:dbscheme", + actual = "//cpp/ql/lib:dbscheme", ) pkg_filegroup( name = "db-files", srcs = [ ":dbscheme", - "@ql//cpp/downgrades", - "@ql//cpp/ql/lib:dbscheme-stats", + "//cpp/downgrades", + "//cpp/ql/lib:dbscheme-stats", ], ) diff --git a/misc/bazel/BUILD.toolchain.tpl b/misc/bazel/BUILD.toolchain.tpl deleted file mode 100644 index 849508908d86..000000000000 --- a/misc/bazel/BUILD.toolchain.tpl +++ /dev/null @@ -1,12 +0,0 @@ -load("@//misc/bazel:toolchain.bzl", "codeql_cli_toolchain") - -codeql_cli_toolchain( - name = "codeql-cli", - path = "{codeql_cli_path}", -) - -toolchain( - name = "codeql-cli-toolchain", - toolchain = ":codeql-cli", - toolchain_type = "@//:toolchain_type", -) diff --git a/misc/bazel/source_dir.bzl.tpl b/misc/bazel/source_dir.bzl.tpl deleted file mode 100644 index 245ccf6e8b8e..000000000000 --- a/misc/bazel/source_dir.bzl.tpl +++ /dev/null @@ -1,11 +0,0 @@ -def root_source_dir(): - """ get absolute path to the root source directory - - This can break hermeticity if used in a build step""" - return '{root}' - -def current_source_dir(): - """ get absolute path to the source directory of this bazel package - - This can break hermeticity if used in a build step""" - return root_source_dir() + '/' + native.package_name() diff --git a/misc/bazel/test.bzl b/misc/bazel/test.bzl deleted file mode 100644 index 24fe7dbe396b..000000000000 --- a/misc/bazel/test.bzl +++ /dev/null @@ -1,38 +0,0 @@ -def _test_script_impl(ctx): - output = ctx.actions.declare_file("%s.py" % ctx.label.name) - codeql_cli_path = ctx.toolchains["//:toolchain_type"].codeql_cli.path - ctx.actions.expand_template( - template = ctx.file._template, - output = output, - substitutions = { - "{codeql_cli_path}": codeql_cli_path, - "{test_sources}": str([f.path for f in ctx.files.srcs]), - }, - ) - return DefaultInfo( - files = depset([output]), - ) - -_test_script = rule( - implementation = _test_script_impl, - attrs = { - "srcs": attr.label_list(allow_files = True), - "_template": attr.label(default = "//misc/bazel:test.template.py", allow_single_file = True), - }, - toolchains = ["//:toolchain_type"], -) - -def codeql_test(*, name, srcs, deps): - srcs = native.glob(["test/**/*.ql", "test/**/*.qlref"]) - data = srcs + deps - script = name + "-script" - _test_script( - name = script, - srcs = srcs, - ) - native.py_test( - name = name, - main = script + ".py", - srcs = [script], - data = data, - ) diff --git a/misc/bazel/test.template.py b/misc/bazel/test.template.py deleted file mode 100644 index 2430afb04b03..000000000000 --- a/misc/bazel/test.template.py +++ /dev/null @@ -1,3 +0,0 @@ -import os - -os.execl("{codeql_cli_path}", "test", "run", "--check-databases", "--", *{test_sources}) diff --git a/misc/bazel/workspace.bzl b/misc/bazel/workspace.bzl index 1b52c486925f..b306363d7c09 100644 --- a/misc/bazel/workspace.bzl +++ b/misc/bazel/workspace.bzl @@ -1,22 +1,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -def _ql_utils_impl(repository_ctx): - root = repository_ctx.path(Label("//:WORKSPACE.bazel")).realpath.dirname - repository_ctx.file("BUILD.bazel") - repository_ctx.template( - "source_dir.bzl", - Label("@ql//misc/bazel:source_dir.bzl.tpl"), - substitutions = {"{root}": str(root)}, - ) - -_ql_utils = repository_rule( - implementation = _ql_utils_impl, -) - def ql_workspace(): - _ql_utils(name = "ql_utils") - maybe( repo_rule = http_archive, name = "rules_pkg", diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 134a32d86037..fd5e5b46de41 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -1,7 +1,6 @@ load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("@rules_pkg//:install.bzl", "pkg_install") -load("@ql//:defs.bzl", "codeql_platform") -load("@ql_utils//:source_dir.bzl", "current_source_dir") +load("//:defs.bzl", "codeql_platform") pkg_files( name = "dbscheme", @@ -56,10 +55,13 @@ pkg_filegroup( ) pkg_install( - name = "create-extractor-pack", + name = "_create_extractor_pack", srcs = ["//swift:extractor-pack"], - args = [ - "--destdir", - current_source_dir() + "/extractor-pack", - ], +) + +py_binary( + name = "create-extractor-pack", + srcs = ["tools/create_extractor_pack.py"], + main = "tools/create_extractor_pack.py", + deps = [":_create_extractor_pack"], ) diff --git a/swift/README.md b/swift/README.md index d2b56d1a6f37..ef805a8273ba 100644 --- a/swift/README.md +++ b/swift/README.md @@ -6,4 +6,4 @@ The Swift codeql package is an experimental and unsupported work in progress. Run `bazel run //swift:create-extractor-pack`, which will install `swift/extractor-pack`. Using `--search-path=swift/extractor-pack` will then pick up the Swift extractor. You can also use -`--search-path=swift`, as the extractor pack is mentioned in `swift/.codeqlmanifest`. +`--search-path=swift`, as the extractor pack is mentioned in `swift/.codeqlmanifest.json`. diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index b201c993c43b..5a87a0b2c150 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -1,5 +1,10 @@ cc_binary( name = "extractor", srcs = ["main.cpp"], + target_compatible_with = select({ + "@platforms//os:linux": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], + }), visibility = ["//swift:__pkg__"], ) diff --git a/swift/ql/test/qlpack.yml b/swift/ql/test/qlpack.yml index 1b025e1c301f..de89e5c9ef81 100644 --- a/swift/ql/test/qlpack.yml +++ b/swift/ql/test/qlpack.yml @@ -1,5 +1,6 @@ name: codeql-swift-tests version: 0.0.0 -libraryPathDependencies: - - codeql/swift-all +dependencies: + codeql/swift-all: "*" +tests: . extractor: swift diff --git a/swift/tools/create_extractor_pack.py b/swift/tools/create_extractor_pack.py new file mode 100644 index 000000000000..9ebf95e31e17 --- /dev/null +++ b/swift/tools/create_extractor_pack.py @@ -0,0 +1,16 @@ +import os +import pathlib +import shutil +import sys +from swift._create_extractor_pack_install_script import main + +try: + workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) +except KeyError: + print("this should be run with bazel run", file=sys.stderr) + sys.exit(1) + +dest_dir = workspace_dir / 'swift' / 'extractor-pack' +shutil.rmtree(dest_dir) +os.environ['DESTDIR'] = str(dest_dir) +main(sys.argv) From 8ef28787b6a119db074b9083c3b46e055a98cc1f Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 12 Apr 2022 17:05:26 +0200 Subject: [PATCH 7/7] Swift: do not fail pack creation if dir does not exist --- swift/tools/create_extractor_pack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/tools/create_extractor_pack.py b/swift/tools/create_extractor_pack.py index 9ebf95e31e17..ce86a00ea2d4 100644 --- a/swift/tools/create_extractor_pack.py +++ b/swift/tools/create_extractor_pack.py @@ -11,6 +11,6 @@ sys.exit(1) dest_dir = workspace_dir / 'swift' / 'extractor-pack' -shutil.rmtree(dest_dir) +shutil.rmtree(dest_dir, ignore_errors=True) os.environ['DESTDIR'] = str(dest_dir) main(sys.argv)