From 24eb65692f616cb1b3f1f8f0b5bad6bd10071ceb Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 26 Nov 2024 15:57:18 +0100 Subject: [PATCH 01/10] Rust: add some performance diagnostics This outputs some duration counts for various parts of the extraction process in the database in the form of telemetry diagnostics. The diagnostics format was preferred to putting things in the relational database as that will scale better to code scanning and is more flexible as for the data we can put into it without passing through the dbscheme. Also, although it's not the case yet, it will be possible to output diagnostics even if creation of the database fails. --- Cargo.lock | 6 +- MODULE.bazel | 2 +- .../tree_sitter_extractors_deps/BUILD.bazel | 2 +- .../BUILD.cargo_metadata-0.18.1.bazel | 2 +- .../BUILD.ra_ap_proc_macro_api-0.0.232.bazel | 2 +- .../BUILD.ra_ap_project_model-0.0.232.bazel | 2 +- ...2.bazel => BUILD.serde_json-1.0.133.bazel} | 6 +- .../tree_sitter_extractors_deps/defs.bzl | 16 +- rust/extractor/Cargo.toml | 2 + rust/extractor/src/config.rs | 1 + rust/extractor/src/diagnostics.rs | 258 ++++++++++++++++++ rust/extractor/src/main.rs | 90 ++++-- rust/ql/integration-tests/conftest.py | 9 + .../hello-project/diagnostics.expected | 101 +++++++ .../hello-project/test_project.py | 4 +- .../diagnostics.cargo.expected | 97 +++++++ .../diagnostics.rust-project.expected | 93 +++++++ .../hello-workspace/test_workspace.py | 6 +- 18 files changed, 656 insertions(+), 43 deletions(-) rename misc/bazel/3rdparty/tree_sitter_extractors_deps/{BUILD.serde_json-1.0.132.bazel => BUILD.serde_json-1.0.133.bazel} (97%) create mode 100644 rust/extractor/src/diagnostics.rs create mode 100644 rust/ql/integration-tests/hello-project/diagnostics.expected create mode 100644 rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected create mode 100644 rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected diff --git a/Cargo.lock b/Cargo.lock index 1d5b8824c84a..b0af4387f2ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -381,6 +381,7 @@ version = "0.1.0" dependencies = [ "anyhow", "argfile", + "chrono", "clap", "codeql-extractor", "figment", @@ -404,6 +405,7 @@ dependencies = [ "ra_ap_vfs", "rust-extractor-macros", "serde", + "serde_json", "serde_with", "stderrlog", "triomphe", @@ -2034,9 +2036,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", diff --git a/MODULE.bazel b/MODULE.bazel index 13c801520b04..311d09088323 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -68,7 +68,7 @@ use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2. # deps for ruby+rust # keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh` tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r") -use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.132", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1") +use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.133", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1") dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet") dotnet.toolchain(dotnet_version = "9.0.100") diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel index 844d385f8a41..6208d66c5d93 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel @@ -243,7 +243,7 @@ alias( alias( name = "serde_json", - actual = "@vendor__serde_json-1.0.132//:serde_json", + actual = "@vendor__serde_json-1.0.133//:serde_json", tags = ["manual"], ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cargo_metadata-0.18.1.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cargo_metadata-0.18.1.bazel index e31c61cfac73..9ad331aa5f74 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cargo_metadata-0.18.1.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cargo_metadata-0.18.1.bazel @@ -86,7 +86,7 @@ rust_library( "@vendor__cargo-platform-0.1.8//:cargo_platform", "@vendor__semver-1.0.23//:semver", "@vendor__serde-1.0.214//:serde", - "@vendor__serde_json-1.0.132//:serde_json", + "@vendor__serde_json-1.0.133//:serde_json", "@vendor__thiserror-1.0.69//:thiserror", ], ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_proc_macro_api-0.0.232.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_proc_macro_api-0.0.232.bazel index d1f5d480d719..fa3b71f570cd 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_proc_macro_api-0.0.232.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_proc_macro_api-0.0.232.bazel @@ -96,7 +96,7 @@ rust_library( "@vendor__ra_ap_tt-0.0.232//:ra_ap_tt", "@vendor__rustc-hash-1.1.0//:rustc_hash", "@vendor__serde-1.0.214//:serde", - "@vendor__serde_json-1.0.132//:serde_json", + "@vendor__serde_json-1.0.133//:serde_json", "@vendor__tracing-0.1.40//:tracing", ], ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.232.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.232.bazel index 6f6f3bb303a4..5608e6dd6b0d 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.232.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.232.bazel @@ -102,7 +102,7 @@ rust_library( "@vendor__rustc-hash-1.1.0//:rustc_hash", "@vendor__semver-1.0.23//:semver", "@vendor__serde-1.0.214//:serde", - "@vendor__serde_json-1.0.132//:serde_json", + "@vendor__serde_json-1.0.133//:serde_json", "@vendor__tracing-0.1.40//:tracing", "@vendor__triomphe-0.1.14//:triomphe", ], diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.132.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.133.bazel similarity index 97% rename from misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.132.bazel rename to misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.133.bazel index ecbae32eeafa..710772e4f37b 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.132.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.133.bazel @@ -83,13 +83,13 @@ rust_library( "@rules_rust//rust/platform:x86_64-unknown-none": [], "//conditions:default": ["@platforms//:incompatible"], }), - version = "1.0.132", + version = "1.0.133", deps = [ "@vendor__itoa-1.0.11//:itoa", "@vendor__memchr-2.7.4//:memchr", "@vendor__ryu-1.0.18//:ryu", "@vendor__serde-1.0.214//:serde", - "@vendor__serde_json-1.0.132//:build_script_build", + "@vendor__serde_json-1.0.133//:build_script_build", ], ) @@ -143,7 +143,7 @@ cargo_build_script( "noclippy", "norustfmt", ], - version = "1.0.132", + version = "1.0.133", visibility = ["//visibility:private"], ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index 1d0b825c2356..927736b56b21 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -320,6 +320,7 @@ _NORMAL_DEPENDENCIES = { _COMMON_CONDITION: { "anyhow": Label("@vendor__anyhow-1.0.93//:anyhow"), "argfile": Label("@vendor__argfile-0.2.1//:argfile"), + "chrono": Label("@vendor__chrono-0.4.38//:chrono"), "clap": Label("@vendor__clap-4.5.20//:clap"), "figment": Label("@vendor__figment-0.10.19//:figment"), "glob": Label("@vendor__glob-0.3.1//:glob"), @@ -341,6 +342,7 @@ _NORMAL_DEPENDENCIES = { "ra_ap_syntax": Label("@vendor__ra_ap_syntax-0.0.232//:ra_ap_syntax"), "ra_ap_vfs": Label("@vendor__ra_ap_vfs-0.0.232//:ra_ap_vfs"), "serde": Label("@vendor__serde-1.0.214//:serde"), + "serde_json": Label("@vendor__serde_json-1.0.133//:serde_json"), "serde_with": Label("@vendor__serde_with-3.11.0//:serde_with"), "stderrlog": Label("@vendor__stderrlog-0.6.0//:stderrlog"), "triomphe": Label("@vendor__triomphe-0.1.14//:triomphe"), @@ -363,7 +365,7 @@ _NORMAL_DEPENDENCIES = { "rayon": Label("@vendor__rayon-1.10.0//:rayon"), "regex": Label("@vendor__regex-1.11.1//:regex"), "serde": Label("@vendor__serde-1.0.214//:serde"), - "serde_json": Label("@vendor__serde_json-1.0.132//:serde_json"), + "serde_json": Label("@vendor__serde_json-1.0.133//:serde_json"), "tracing": Label("@vendor__tracing-0.1.40//:tracing"), "tracing-subscriber": Label("@vendor__tracing-subscriber-0.3.18//:tracing_subscriber"), "tree-sitter": Label("@vendor__tree-sitter-0.24.4//:tree_sitter"), @@ -2508,12 +2510,12 @@ def crate_repositories(): maybe( http_archive, - name = "vendor__serde_json-1.0.132", - sha256 = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03", + name = "vendor__serde_json-1.0.133", + sha256 = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377", type = "tar.gz", - urls = ["https://static.crates.io/crates/serde_json/1.0.132/download"], - strip_prefix = "serde_json-1.0.132", - build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.serde_json-1.0.132.bazel"), + urls = ["https://static.crates.io/crates/serde_json/1.0.133/download"], + strip_prefix = "serde_json-1.0.133", + build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.serde_json-1.0.133.bazel"), ) maybe( @@ -3341,7 +3343,7 @@ def crate_repositories(): struct(repo = "vendor__rayon-1.10.0", is_dev_dep = False), struct(repo = "vendor__regex-1.11.1", is_dev_dep = False), struct(repo = "vendor__serde-1.0.214", is_dev_dep = False), - struct(repo = "vendor__serde_json-1.0.132", is_dev_dep = False), + struct(repo = "vendor__serde_json-1.0.133", is_dev_dep = False), struct(repo = "vendor__serde_with-3.11.0", is_dev_dep = False), struct(repo = "vendor__stderrlog-0.6.0", is_dev_dep = False), struct(repo = "vendor__syn-2.0.87", is_dev_dep = False), diff --git a/rust/extractor/Cargo.toml b/rust/extractor/Cargo.toml index 8b58898d3cf3..591df3480cd3 100644 --- a/rust/extractor/Cargo.toml +++ b/rust/extractor/Cargo.toml @@ -33,3 +33,5 @@ codeql-extractor = { path = "../../shared/tree-sitter-extractor" } rust-extractor-macros = { path = "macros" } itertools = "0.13.0" glob = "0.3.1" +chrono = { version = "0.4.38", features = ["serde"] } +serde_json = "1.0.133" diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 70c390b99491..0e92e82a58cb 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -45,6 +45,7 @@ pub struct Config { pub scratch_dir: PathBuf, pub trap_dir: PathBuf, pub source_archive_dir: PathBuf, + pub diagnostic_dir: PathBuf, pub cargo_target_dir: Option, pub cargo_target: Option, pub cargo_features: Vec, diff --git a/rust/extractor/src/diagnostics.rs b/rust/extractor/src/diagnostics.rs new file mode 100644 index 000000000000..21b3619a207d --- /dev/null +++ b/rust/extractor/src/diagnostics.rs @@ -0,0 +1,258 @@ +use crate::config::Config; +use anyhow::Context; +use chrono::{DateTime, Utc}; +use log::{debug, info}; +use ra_ap_project_model::ProjectManifest; +use serde::Serialize; +use std::fmt::Display; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +#[derive(Default, Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(dead_code)] +enum Severity { + #[default] + Note, + Warning, + Error, +} + +#[derive(Default, Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +struct Visibility { + status_page: bool, + cli_summary_table: bool, + telemetry: bool, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(dead_code)] +enum Message { + TextMessage(String), + MarkdownMessage(String), +} + +impl Default for Message { + fn default() -> Self { + Message::TextMessage("".to_string()) + } +} + +#[derive(Default, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct Source { + id: String, + name: String, + extractor_name: String, +} + +#[derive(Default, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct Location { + file: PathBuf, + start_line: u32, + start_column: u32, + end_line: u32, + end_column: u32, +} + +#[derive(Default, Debug, Clone, Serialize)] +pub struct Diagnostics { + source: Source, + visibility: Visibility, + severity: Severity, + #[serde(flatten)] + message: Message, + timestamp: DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + location: Option, + attributes: T, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +enum ExtractionStepTarget { + LoadManifest(PathBuf), + FetchFile(PathBuf), + Parse(PathBuf), + Extract(PathBuf), +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ExtractionStep { + #[serde(flatten)] + target: ExtractionStepTarget, + ms: u128, +} + +impl ExtractionStep { + fn new(start: Instant, target: ExtractionStepTarget) -> Self { + let ret = ExtractionStep { + target, + ms: start.elapsed().as_millis(), + }; + debug!("{ret:?}"); + ret + } + + pub fn load_manifest(start: Instant, target: &ProjectManifest) -> Self { + Self::new( + start, + ExtractionStepTarget::LoadManifest(PathBuf::from(target.manifest_path())), + ) + } + + pub fn parse(start: Instant, target: &Path) -> Self { + Self::new(start, ExtractionStepTarget::Parse(PathBuf::from(target))) + } + + pub fn extract(start: Instant, target: &Path) -> Self { + Self::new(start, ExtractionStepTarget::Extract(PathBuf::from(target))) + } + + pub fn fetch_file(start: Instant, target: &Path) -> Self { + Self::new( + start, + ExtractionStepTarget::FetchFile(PathBuf::from(target)), + ) + } +} + +#[derive(Debug, Default, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct HumanReadableDuration { + ms: u128, + pretty: String, +} + +impl HumanReadableDuration { + pub fn new(ms: u128) -> Self { + let seconds = ms / 1000; + let minutes = seconds / 60; + let hours = minutes / 60; + let pretty = format!( + "{hours}:{minutes:02}:{seconds:02}.{milliseconds:03}", + minutes = minutes % 60, + seconds = seconds % 60, + milliseconds = ms % 1000, + ); + Self { ms, pretty } + } +} + +impl From for HumanReadableDuration { + fn from(val: u128) -> Self { + HumanReadableDuration::new(val) + } +} + +impl Display for HumanReadableDuration { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}ms ({})", self.ms, self.pretty) + } +} + +#[derive(Debug, Default, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ExtractionSummary { + number_of_manifests: usize, + number_of_files: usize, + total_load_duration: HumanReadableDuration, + total_fetch_file_duration: HumanReadableDuration, + total_parse_duration: HumanReadableDuration, + total_extract_duration: HumanReadableDuration, + total_duration: HumanReadableDuration, +} + +#[derive(Debug, Default, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ExtractionAttributes { + steps: Vec, + summary: ExtractionSummary, +} + +type ExtractionDiagnostics = Diagnostics; + +fn summary(start: Instant, steps: &[ExtractionStep]) -> ExtractionSummary { + let mut number_of_manifests = 0; + let mut number_of_files = 0; + let mut total_load_duration = 0; + let mut total_parse_duration = 0; + let mut total_extract_duration = 0; + let mut total_fetch_file_duration: u128 = 0; + for step in steps { + match &step.target { + ExtractionStepTarget::LoadManifest(_) => { + number_of_manifests += 1; + total_load_duration += step.ms; + } + ExtractionStepTarget::FetchFile(_) => { + number_of_files += 1; + total_fetch_file_duration += step.ms; + } + ExtractionStepTarget::Parse(_) => { + total_parse_duration += step.ms; + } + ExtractionStepTarget::Extract(_) => { + total_extract_duration += step.ms; + } + } + } + let ret = ExtractionSummary { + number_of_manifests, + number_of_files, + total_load_duration: total_load_duration.into(), + total_fetch_file_duration: total_fetch_file_duration.into(), + total_parse_duration: total_parse_duration.into(), + total_extract_duration: total_extract_duration.into(), + total_duration: start.elapsed().as_millis().into(), + }; + info!("total loadimg duration: {}", ret.total_load_duration); + info!( + "total file fetching duration: {}", + ret.total_fetch_file_duration + ); + info!("total parsing duration: {}", ret.total_parse_duration); + info!("total extracting duration: {}", ret.total_extract_duration); + info!("total duration: {}", ret.total_duration); + ret +} + +pub fn emit_extraction_diagnostics( + start: Instant, + config: &Config, + steps: Vec, +) -> anyhow::Result<()> { + let summary = summary(start, &steps); + let diagnostics = ExtractionDiagnostics { + source: Source { + id: "rust/extractor/telemetry".to_owned(), + name: "telemetry".to_string(), + extractor_name: "rust".to_string(), + }, + visibility: Visibility { + telemetry: true, + ..Default::default() + }, + timestamp: Utc::now(), + attributes: ExtractionAttributes { steps, summary }, + ..Default::default() + }; + + std::fs::create_dir_all(&config.diagnostic_dir).with_context(|| { + format!( + "creating diagnostics directory {}", + config.diagnostic_dir.display() + ) + })?; + let target = config.diagnostic_dir.join("extraction.jsonc"); + let mut output = File::create(&target) + .with_context(|| format!("creating diagnostics file {}", target.display()))?; + serde_json::to_writer_pretty(&mut output, &diagnostics) + .with_context(|| format!("writing to diagnostics file {}", target.display()))?; + Ok(()) +} diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index ecbdc965f8e7..f00e94e9ba4e 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -1,3 +1,4 @@ +use crate::diagnostics::{emit_extraction_diagnostics, ExtractionStep}; use crate::rust_analyzer::path_to_file_id; use anyhow::Context; use archive::Archiver; @@ -5,9 +6,10 @@ use log::info; use ra_ap_hir::Semantics; use ra_ap_ide_db::line_index::{LineCol, LineIndex}; use ra_ap_ide_db::RootDatabase; -use ra_ap_project_model::ProjectManifest; +use ra_ap_project_model::{CargoConfig, ProjectManifest}; use ra_ap_vfs::Vfs; use rust_analyzer::{ParseResult, RustAnalyzer}; +use std::time::Instant; use std::{ collections::HashMap, path::{Path, PathBuf}, @@ -15,6 +17,7 @@ use std::{ mod archive; mod config; +mod diagnostics; pub mod generated; mod qltest; mod rust_analyzer; @@ -24,18 +27,31 @@ pub mod trap; struct Extractor<'a> { archiver: &'a Archiver, traps: &'a trap::TrapFileProvider, + steps: Vec, } -impl Extractor<'_> { - fn extract(&self, rust_analyzer: &rust_analyzer::RustAnalyzer, file: &std::path::Path) { +impl<'a> Extractor<'a> { + pub fn new(archiver: &'a Archiver, traps: &'a trap::TrapFileProvider) -> Self { + Self { + archiver, + traps, + steps: Vec::new(), + } + } + + fn extract(&mut self, rust_analyzer: &rust_analyzer::RustAnalyzer, file: &std::path::Path) { self.archiver.archive(file); + let before_parse = Instant::now(); let ParseResult { ast, text, errors, semantics_info, } = rust_analyzer.parse(file); + self.steps.push(ExtractionStep::parse(before_parse, file)); + + let before_extract = Instant::now(); let line_index = LineIndex::new(text.as_ref()); let display_path = file.to_string_lossy(); let mut trap = self.traps.create("source", file); @@ -73,22 +89,63 @@ impl Extractor<'_> { err.to_string() ) }); + self.steps + .push(ExtractionStep::extract(before_extract, file)); } pub fn extract_with_semantics( - &self, + &mut self, file: &Path, semantics: &Semantics<'_, RootDatabase>, vfs: &Vfs, ) { self.extract(&RustAnalyzer::new(vfs, semantics), file); } - pub fn extract_without_semantics(&self, file: &Path, reason: &str) { + + pub fn extract_without_semantics(&mut self, file: &Path, reason: &str) { self.extract(&RustAnalyzer::WithoutSemantics { reason }, file); } + + pub fn load_manifest( + &mut self, + project: &ProjectManifest, + config: &CargoConfig, + ) -> Option<(RootDatabase, Vfs)> { + let before = Instant::now(); + let ret = RustAnalyzer::load_workspace(project, config); + self.steps + .push(ExtractionStep::load_manifest(before, project)); + ret + } + + pub fn fetch_file( + &mut self, + file: &Path, + semantics: &Semantics<'_, RootDatabase>, + vfs: &Vfs, + ) -> Result<(), String> { + let before = Instant::now(); + let Some(id) = path_to_file_id(file, vfs) else { + return Err("not included in files loaded from manifest".to_string()); + }; + if semantics.file_to_module_def(id).is_none() { + return Err("not included as a module".to_string()); + } + self.steps.push(ExtractionStep::fetch_file(before, file)); + Ok(()) + } + + pub fn emit_extraction_diagnostics( + self, + start: Instant, + cfg: &config::Config, + ) -> anyhow::Result<()> { + emit_extraction_diagnostics(start, cfg, self.steps) + } } fn main() -> anyhow::Result<()> { + let start = Instant::now(); let mut cfg = config::Config::extract().context("failed to load configuration")?; stderrlog::new() .module(module_path!()) @@ -103,10 +160,7 @@ fn main() -> anyhow::Result<()> { let archiver = archive::Archiver { root: cfg.source_archive_dir.clone(), }; - let extractor = Extractor { - archiver: &archiver, - traps: &traps, - }; + let mut extractor = Extractor::new(&archiver, &traps); let files: Vec = cfg .inputs .iter() @@ -132,21 +186,13 @@ fn main() -> anyhow::Result<()> { } let cargo_config = cfg.to_cargo_config(); for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) { - if let Some((ref db, ref vfs)) = RustAnalyzer::load_workspace(manifest, &cargo_config) { + if let Some((ref db, ref vfs)) = extractor.load_manifest(manifest, &cargo_config) { let semantics = Semantics::new(db); for file in files { - let Some(id) = path_to_file_id(file, vfs) else { - extractor.extract_without_semantics( - file, - "not included in files loaded from manifest", - ); - continue; + match extractor.fetch_file(file, &semantics, vfs) { + Ok(()) => extractor.extract_with_semantics(file, &semantics, vfs), + Err(reason) => extractor.extract_without_semantics(file, &reason), }; - if semantics.file_to_module_def(id).is_none() { - extractor.extract_without_semantics(file, "not included as a module"); - continue; - } - extractor.extract_with_semantics(file, &semantics, vfs); } } else { for file in files { @@ -155,5 +201,5 @@ fn main() -> anyhow::Result<()> { } } - Ok(()) + extractor.emit_extraction_diagnostics(start, &cfg) } diff --git a/rust/ql/integration-tests/conftest.py b/rust/ql/integration-tests/conftest.py index 08b17f106f80..9967339f2d2d 100644 --- a/rust/ql/integration-tests/conftest.py +++ b/rust/ql/integration-tests/conftest.py @@ -13,3 +13,12 @@ def select(self, name: str): @pytest.fixture def manifests(cwd): return _Manifests(cwd) + +@pytest.fixture +def rust_check_diagnostics(check_diagnostics): + check_diagnostics.replacements += [ + (r'"ms"\s*:\s*[0-9]+', '"ms": "REDACTED"'), + (r'"pretty"\s*:\s*"[0-9]+:[0-9]{2}:[0-9]{2}.[0-9]{3}"', '"pretty": "REDACTED"'), + (r'Cargo.toml|rust-project.json', ""), + ] + return check_diagnostics diff --git a/rust/ql/integration-tests/hello-project/diagnostics.expected b/rust/ql/integration-tests/hello-project/diagnostics.expected new file mode 100644 index 000000000000..165fd2639901 --- /dev/null +++ b/rust/ql/integration-tests/hello-project/diagnostics.expected @@ -0,0 +1,101 @@ +{ + "attributes": { + "steps": [ + { + "loadManifest": "/", + "ms": "REDACTED" + }, + { + "fetchFile": "/src/directory_module/mod.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/src/directory_module/mod.rs" + }, + { + "extract": "/src/directory_module/mod.rs", + "ms": "REDACTED" + }, + { + "fetchFile": "/src/directory_module/nested_module.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/src/directory_module/nested_module.rs" + }, + { + "extract": "/src/directory_module/nested_module.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/src/directory_module/not_loaded.rs" + }, + { + "extract": "/src/directory_module/not_loaded.rs", + "ms": "REDACTED" + }, + { + "fetchFile": "/src/file_module.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/src/file_module.rs" + }, + { + "extract": "/src/file_module.rs", + "ms": "REDACTED" + }, + { + "fetchFile": "/src/main.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/src/main.rs" + }, + { + "extract": "/src/main.rs", + "ms": "REDACTED" + } + ], + "summary": { + "numberOfFiles": 4, + "numberOfManifests": 1, + "totalDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalExtractDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalFetchFileDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalLoadDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalParseDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + } + } + }, + "severity": "note", + "source": { + "extractorName": "rust", + "id": "rust/extractor/telemetry", + "name": "telemetry" + }, + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/rust/ql/integration-tests/hello-project/test_project.py b/rust/ql/integration-tests/hello-project/test_project.py index d03c4f67e265..2cbac0ffdb8a 100644 --- a/rust/ql/integration-tests/hello-project/test_project.py +++ b/rust/ql/integration-tests/hello-project/test_project.py @@ -1,7 +1,7 @@ -def test_cargo(codeql, rust, manifests, check_source_archive): +def test_cargo(codeql, rust, manifests, check_source_archive, rust_check_diagnostics): manifests.select("Cargo.toml") codeql.database.create() -def test_rust_project(codeql, rust, manifests, check_source_archive): +def test_rust_project(codeql, rust, manifests, check_source_archive, rust_check_diagnostics): manifests.select("rust-project.json") codeql.database.create() diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected new file mode 100644 index 000000000000..42e7859a4e82 --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected @@ -0,0 +1,97 @@ +{ + "attributes": { + "steps": [ + { + "loadManifest": "/lib/", + "ms": "REDACTED" + }, + { + "fetchFile": "/lib/src/a_module/mod.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/lib/src/a_module/mod.rs" + }, + { + "extract": "/lib/src/a_module/mod.rs", + "ms": "REDACTED" + }, + { + "fetchFile": "/lib/src/lib.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/lib/src/lib.rs" + }, + { + "extract": "/lib/src/lib.rs", + "ms": "REDACTED" + }, + { + "loadManifest": "/exe/", + "ms": "REDACTED" + }, + { + "fetchFile": "/exe/src/a_module.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/exe/src/a_module.rs" + }, + { + "extract": "/exe/src/a_module.rs", + "ms": "REDACTED" + }, + { + "fetchFile": "/exe/src/main.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/exe/src/main.rs" + }, + { + "extract": "/exe/src/main.rs", + "ms": "REDACTED" + } + ], + "summary": { + "numberOfFiles": 4, + "numberOfManifests": 2, + "totalDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalExtractDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalFetchFileDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalLoadDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalParseDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + } + } + }, + "severity": "note", + "source": { + "extractorName": "rust", + "id": "rust/extractor/telemetry", + "name": "telemetry" + }, + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected new file mode 100644 index 000000000000..7df7b52cdee0 --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected @@ -0,0 +1,93 @@ +{ + "attributes": { + "steps": [ + { + "loadManifest": "/", + "ms": "REDACTED" + }, + { + "fetchFile": "/exe/src/a_module.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/exe/src/a_module.rs" + }, + { + "extract": "/exe/src/a_module.rs", + "ms": "REDACTED" + }, + { + "fetchFile": "/exe/src/main.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/exe/src/main.rs" + }, + { + "extract": "/exe/src/main.rs", + "ms": "REDACTED" + }, + { + "fetchFile": "/lib/src/a_module/mod.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/lib/src/a_module/mod.rs" + }, + { + "extract": "/lib/src/a_module/mod.rs", + "ms": "REDACTED" + }, + { + "fetchFile": "/lib/src/lib.rs", + "ms": "REDACTED" + }, + { + "ms": "REDACTED", + "parse": "/lib/src/lib.rs" + }, + { + "extract": "/lib/src/lib.rs", + "ms": "REDACTED" + } + ], + "summary": { + "numberOfFiles": 4, + "numberOfManifests": 1, + "totalDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalExtractDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalFetchFileDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalLoadDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + }, + "totalParseDuration": { + "ms": "REDACTED", + "pretty": "REDACTED" + } + } + }, + "severity": "note", + "source": { + "extractorName": "rust", + "id": "rust/extractor/telemetry", + "name": "telemetry" + }, + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/rust/ql/integration-tests/hello-workspace/test_workspace.py b/rust/ql/integration-tests/hello-workspace/test_workspace.py index f3503e7cefae..5c95031466f8 100644 --- a/rust/ql/integration-tests/hello-workspace/test_workspace.py +++ b/rust/ql/integration-tests/hello-workspace/test_workspace.py @@ -3,10 +3,12 @@ # currently the DB-check fails on actions because of loading files multiple times and assiging multiple locations # see https://github.com/github/codeql-team/issues/3365 @pytest.mark.ql_test("DB-CHECK", xfail="maybe") -def test_cargo(codeql, rust, manifests, check_source_archive): +def test_cargo(codeql, rust, manifests, check_source_archive, rust_check_diagnostics): + rust_check_diagnostics.expected_suffix = ".cargo.expected" manifests.select("Cargo.toml") codeql.database.create() -def test_rust_project(codeql, rust, manifests, check_source_archive): +def test_rust_project(codeql, rust, manifests, check_source_archive, rust_check_diagnostics): + rust_check_diagnostics.expected_suffix = ".rust-project.expected" manifests.select("rust-project.json") codeql.database.create() From 8abd3c4707b527cf70e05f29eea88cb4ca608a2b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 26 Nov 2024 16:48:49 +0100 Subject: [PATCH 02/10] Rust: Remove windows difference from diagnostics --- rust/ql/integration-tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/ql/integration-tests/conftest.py b/rust/ql/integration-tests/conftest.py index 9967339f2d2d..13cecd478d7b 100644 --- a/rust/ql/integration-tests/conftest.py +++ b/rust/ql/integration-tests/conftest.py @@ -20,5 +20,6 @@ def rust_check_diagnostics(check_diagnostics): (r'"ms"\s*:\s*[0-9]+', '"ms": "REDACTED"'), (r'"pretty"\s*:\s*"[0-9]+:[0-9]{2}:[0-9]{2}.[0-9]{3}"', '"pretty": "REDACTED"'), (r'Cargo.toml|rust-project.json', ""), + (r'"//\?/', '"'), # remove windows `//?/` long path syntax ] return check_diagnostics From 556774edc7d68546ee41f7738f80f50db4502ce9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 26 Nov 2024 18:00:15 +0100 Subject: [PATCH 03/10] Rust: do not put extraction steps in the expected diagnostics --- rust/ql/integration-tests/conftest.py | 5 +- .../hello-project/diagnostics.expected | 62 ------------------- .../diagnostics.cargo.expected | 58 ----------------- .../diagnostics.rust-project.expected | 54 ---------------- 4 files changed, 3 insertions(+), 176 deletions(-) diff --git a/rust/ql/integration-tests/conftest.py b/rust/ql/integration-tests/conftest.py index 13cecd478d7b..f05b1f5c98ae 100644 --- a/rust/ql/integration-tests/conftest.py +++ b/rust/ql/integration-tests/conftest.py @@ -19,7 +19,8 @@ def rust_check_diagnostics(check_diagnostics): check_diagnostics.replacements += [ (r'"ms"\s*:\s*[0-9]+', '"ms": "REDACTED"'), (r'"pretty"\s*:\s*"[0-9]+:[0-9]{2}:[0-9]{2}.[0-9]{3}"', '"pretty": "REDACTED"'), - (r'Cargo.toml|rust-project.json', ""), - (r'"//\?/', '"'), # remove windows `//?/` long path syntax + ] + check_diagnostics.skip += [ + "attributes.steps", # the order of the steps is not stable ] return check_diagnostics diff --git a/rust/ql/integration-tests/hello-project/diagnostics.expected b/rust/ql/integration-tests/hello-project/diagnostics.expected index 165fd2639901..1c46a0a9f99c 100644 --- a/rust/ql/integration-tests/hello-project/diagnostics.expected +++ b/rust/ql/integration-tests/hello-project/diagnostics.expected @@ -1,67 +1,5 @@ { "attributes": { - "steps": [ - { - "loadManifest": "/", - "ms": "REDACTED" - }, - { - "fetchFile": "/src/directory_module/mod.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/src/directory_module/mod.rs" - }, - { - "extract": "/src/directory_module/mod.rs", - "ms": "REDACTED" - }, - { - "fetchFile": "/src/directory_module/nested_module.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/src/directory_module/nested_module.rs" - }, - { - "extract": "/src/directory_module/nested_module.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/src/directory_module/not_loaded.rs" - }, - { - "extract": "/src/directory_module/not_loaded.rs", - "ms": "REDACTED" - }, - { - "fetchFile": "/src/file_module.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/src/file_module.rs" - }, - { - "extract": "/src/file_module.rs", - "ms": "REDACTED" - }, - { - "fetchFile": "/src/main.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/src/main.rs" - }, - { - "extract": "/src/main.rs", - "ms": "REDACTED" - } - ], "summary": { "numberOfFiles": 4, "numberOfManifests": 1, diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected index 42e7859a4e82..48ae4a51e04f 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected @@ -1,63 +1,5 @@ { "attributes": { - "steps": [ - { - "loadManifest": "/lib/", - "ms": "REDACTED" - }, - { - "fetchFile": "/lib/src/a_module/mod.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/lib/src/a_module/mod.rs" - }, - { - "extract": "/lib/src/a_module/mod.rs", - "ms": "REDACTED" - }, - { - "fetchFile": "/lib/src/lib.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/lib/src/lib.rs" - }, - { - "extract": "/lib/src/lib.rs", - "ms": "REDACTED" - }, - { - "loadManifest": "/exe/", - "ms": "REDACTED" - }, - { - "fetchFile": "/exe/src/a_module.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/exe/src/a_module.rs" - }, - { - "extract": "/exe/src/a_module.rs", - "ms": "REDACTED" - }, - { - "fetchFile": "/exe/src/main.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/exe/src/main.rs" - }, - { - "extract": "/exe/src/main.rs", - "ms": "REDACTED" - } - ], "summary": { "numberOfFiles": 4, "numberOfManifests": 2, diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected index 7df7b52cdee0..1c46a0a9f99c 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected @@ -1,59 +1,5 @@ { "attributes": { - "steps": [ - { - "loadManifest": "/", - "ms": "REDACTED" - }, - { - "fetchFile": "/exe/src/a_module.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/exe/src/a_module.rs" - }, - { - "extract": "/exe/src/a_module.rs", - "ms": "REDACTED" - }, - { - "fetchFile": "/exe/src/main.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/exe/src/main.rs" - }, - { - "extract": "/exe/src/main.rs", - "ms": "REDACTED" - }, - { - "fetchFile": "/lib/src/a_module/mod.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/lib/src/a_module/mod.rs" - }, - { - "extract": "/lib/src/a_module/mod.rs", - "ms": "REDACTED" - }, - { - "fetchFile": "/lib/src/lib.rs", - "ms": "REDACTED" - }, - { - "ms": "REDACTED", - "parse": "/lib/src/lib.rs" - }, - { - "extract": "/lib/src/lib.rs", - "ms": "REDACTED" - } - ], "summary": { "numberOfFiles": 4, "numberOfManifests": 1, From 27738eaaccc54b1ef19e62a3cc6de23eb166cb03 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 27 Nov 2024 12:03:32 +0100 Subject: [PATCH 04/10] Rust: reorganize perf diagnostics --- rust/extractor/src/diagnostics.rs | 146 +++++++++--------- rust/extractor/src/main.rs | 6 +- rust/ql/integration-tests/conftest.py | 4 +- .../hello-project/diagnostics.expected | 20 +-- .../diagnostics.cargo.expected | 44 +++--- .../diagnostics.rust-project.expected | 44 +++--- 6 files changed, 136 insertions(+), 128 deletions(-) diff --git a/rust/extractor/src/diagnostics.rs b/rust/extractor/src/diagnostics.rs index 21b3619a207d..d4b60211bbfe 100644 --- a/rust/extractor/src/diagnostics.rs +++ b/rust/extractor/src/diagnostics.rs @@ -3,7 +3,9 @@ use anyhow::Context; use chrono::{DateTime, Utc}; use log::{debug, info}; use ra_ap_project_model::ProjectManifest; +use serde::ser::SerializeMap; use serde::Serialize; +use std::collections::HashMap; use std::fmt::Display; use std::fs::File; use std::path::{Path, PathBuf}; @@ -72,27 +74,29 @@ pub struct Diagnostics { attributes: T, } -#[derive(Debug, Clone, Serialize)] +#[derive(Default, Debug, Clone, Copy, Serialize, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] -enum ExtractionStepTarget { - LoadManifest(PathBuf), - FetchFile(PathBuf), - Parse(PathBuf), - Extract(PathBuf), +enum ExtractionStepKind { + #[default] + LoadManifest, + LoadSource, + Parse, + Extract, } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct ExtractionStep { - #[serde(flatten)] - target: ExtractionStepTarget, + action: ExtractionStepKind, + file: PathBuf, ms: u128, } impl ExtractionStep { - fn new(start: Instant, target: ExtractionStepTarget) -> Self { + fn new(start: Instant, action: ExtractionStepKind, file: PathBuf) -> Self { let ret = ExtractionStep { - target, + action, + file, ms: start.elapsed().as_millis(), }; debug!("{ret:?}"); @@ -102,70 +106,84 @@ impl ExtractionStep { pub fn load_manifest(start: Instant, target: &ProjectManifest) -> Self { Self::new( start, - ExtractionStepTarget::LoadManifest(PathBuf::from(target.manifest_path())), + ExtractionStepKind::LoadManifest, + PathBuf::from(target.manifest_path()), ) } pub fn parse(start: Instant, target: &Path) -> Self { - Self::new(start, ExtractionStepTarget::Parse(PathBuf::from(target))) + Self::new(start, ExtractionStepKind::Parse, PathBuf::from(target)) } pub fn extract(start: Instant, target: &Path) -> Self { - Self::new(start, ExtractionStepTarget::Extract(PathBuf::from(target))) + Self::new(start, ExtractionStepKind::Extract, PathBuf::from(target)) } - pub fn fetch_file(start: Instant, target: &Path) -> Self { - Self::new( - start, - ExtractionStepTarget::FetchFile(PathBuf::from(target)), - ) + pub fn load_source(start: Instant, target: &Path) -> Self { + Self::new(start, ExtractionStepKind::LoadSource, PathBuf::from(target)) } } -#[derive(Debug, Default, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -struct HumanReadableDuration { - ms: u128, - pretty: String, +#[derive(Debug, Default, Clone)] +struct HumanReadableDuration(u128); + +impl Serialize for HumanReadableDuration { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("ms", &self.0)?; + map.serialize_entry("pretty", &self.pretty())?; + map.end() + } } impl HumanReadableDuration { - pub fn new(ms: u128) -> Self { - let seconds = ms / 1000; - let minutes = seconds / 60; + pub fn add(&mut self, other: u128) { + self.0 += other; + } + + pub fn pretty(&self) -> String { + let milliseconds = self.0 % 1000; + let mut seconds = self.0 / 1000; + if seconds < 60 { + return format!("{seconds}.{milliseconds:03}s"); + } + let mut minutes = seconds / 60; + seconds %= 60; + if minutes < 60 { + return format!("{minutes}min{seconds:02}.{milliseconds:03}s"); + } let hours = minutes / 60; - let pretty = format!( - "{hours}:{minutes:02}:{seconds:02}.{milliseconds:03}", - minutes = minutes % 60, - seconds = seconds % 60, - milliseconds = ms % 1000, - ); - Self { ms, pretty } + minutes %= 60; + format!("{hours}h{minutes:02}min{seconds:02}.{milliseconds:03}s") } } impl From for HumanReadableDuration { fn from(val: u128) -> Self { - HumanReadableDuration::new(val) + HumanReadableDuration(val) } } impl Display for HumanReadableDuration { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}ms ({})", self.ms, self.pretty) + f.write_str(&self.pretty()) } } +#[derive(Debug, Default, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct DurationsSummary { + #[serde(flatten)] + durations: HashMap, + total: HumanReadableDuration, +} + #[derive(Debug, Default, Clone, Serialize)] #[serde(rename_all = "camelCase")] struct ExtractionSummary { number_of_manifests: usize, number_of_files: usize, - total_load_duration: HumanReadableDuration, - total_fetch_file_duration: HumanReadableDuration, - total_parse_duration: HumanReadableDuration, - total_extract_duration: HumanReadableDuration, - total_duration: HumanReadableDuration, + durations: DurationsSummary, } #[derive(Debug, Default, Clone, Serialize)] @@ -180,46 +198,32 @@ type ExtractionDiagnostics = Diagnostics; fn summary(start: Instant, steps: &[ExtractionStep]) -> ExtractionSummary { let mut number_of_manifests = 0; let mut number_of_files = 0; - let mut total_load_duration = 0; - let mut total_parse_duration = 0; - let mut total_extract_duration = 0; - let mut total_fetch_file_duration: u128 = 0; + let mut durations = HashMap::new(); for step in steps { - match &step.target { - ExtractionStepTarget::LoadManifest(_) => { + match &step.action { + ExtractionStepKind::LoadManifest => { number_of_manifests += 1; - total_load_duration += step.ms; } - ExtractionStepTarget::FetchFile(_) => { + ExtractionStepKind::Parse => { number_of_files += 1; - total_fetch_file_duration += step.ms; - } - ExtractionStepTarget::Parse(_) => { - total_parse_duration += step.ms; - } - ExtractionStepTarget::Extract(_) => { - total_extract_duration += step.ms; } + _ => {} } + durations + .entry(step.action) + .or_insert(HumanReadableDuration(0)) + .add(step.ms); } - let ret = ExtractionSummary { + let total = start.elapsed().as_millis().into(); + for (key, value) in &durations { + info!("total duration ({key:?}): {value}"); + } + info!("total duration: {total}"); + ExtractionSummary { number_of_manifests, number_of_files, - total_load_duration: total_load_duration.into(), - total_fetch_file_duration: total_fetch_file_duration.into(), - total_parse_duration: total_parse_duration.into(), - total_extract_duration: total_extract_duration.into(), - total_duration: start.elapsed().as_millis().into(), - }; - info!("total loadimg duration: {}", ret.total_load_duration); - info!( - "total file fetching duration: {}", - ret.total_fetch_file_duration - ); - info!("total parsing duration: {}", ret.total_parse_duration); - info!("total extracting duration: {}", ret.total_extract_duration); - info!("total duration: {}", ret.total_duration); - ret + durations: DurationsSummary { durations, total }, + } } pub fn emit_extraction_diagnostics( diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index f00e94e9ba4e..19239cf9dd48 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -118,7 +118,7 @@ impl<'a> Extractor<'a> { ret } - pub fn fetch_file( + pub fn load_source( &mut self, file: &Path, semantics: &Semantics<'_, RootDatabase>, @@ -131,7 +131,7 @@ impl<'a> Extractor<'a> { if semantics.file_to_module_def(id).is_none() { return Err("not included as a module".to_string()); } - self.steps.push(ExtractionStep::fetch_file(before, file)); + self.steps.push(ExtractionStep::load_source(before, file)); Ok(()) } @@ -189,7 +189,7 @@ fn main() -> anyhow::Result<()> { if let Some((ref db, ref vfs)) = extractor.load_manifest(manifest, &cargo_config) { let semantics = Semantics::new(db); for file in files { - match extractor.fetch_file(file, &semantics, vfs) { + match extractor.load_source(file, &semantics, vfs) { Ok(()) => extractor.extract_with_semantics(file, &semantics, vfs), Err(reason) => extractor.extract_without_semantics(file, &reason), }; diff --git a/rust/ql/integration-tests/conftest.py b/rust/ql/integration-tests/conftest.py index f05b1f5c98ae..895650183c88 100644 --- a/rust/ql/integration-tests/conftest.py +++ b/rust/ql/integration-tests/conftest.py @@ -17,8 +17,8 @@ def manifests(cwd): @pytest.fixture def rust_check_diagnostics(check_diagnostics): check_diagnostics.replacements += [ - (r'"ms"\s*:\s*[0-9]+', '"ms": "REDACTED"'), - (r'"pretty"\s*:\s*"[0-9]+:[0-9]{2}:[0-9]{2}.[0-9]{3}"', '"pretty": "REDACTED"'), + (r'"ms"\s*:\s*[0-9]+', '"ms": "__REDACTED__"'), + (r'"pretty"\s*:\s*"[^"]*"', '"pretty": "__REDACTED__"'), ] check_diagnostics.skip += [ "attributes.steps", # the order of the steps is not stable diff --git a/rust/ql/integration-tests/hello-project/diagnostics.expected b/rust/ql/integration-tests/hello-project/diagnostics.expected index 1c46a0a9f99c..c19e46a88cda 100644 --- a/rust/ql/integration-tests/hello-project/diagnostics.expected +++ b/rust/ql/integration-tests/hello-project/diagnostics.expected @@ -4,24 +4,24 @@ "numberOfFiles": 4, "numberOfManifests": 1, "totalDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" + "ms": "__REDACTED__", + "pretty": "__REDACTED__" }, "totalExtractDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" + "ms": "__REDACTED__", + "pretty": "__REDACTED__" }, "totalFetchFileDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" + "ms": "__REDACTED__", + "pretty": "__REDACTED__" }, "totalLoadDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" + "ms": "__REDACTED__", + "pretty": "__REDACTED__" }, "totalParseDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" + "ms": "__REDACTED__", + "pretty": "__REDACTED__" } } }, diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected index 48ae4a51e04f..052fe3e78393 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected @@ -1,28 +1,30 @@ { "attributes": { "summary": { - "numberOfFiles": 4, - "numberOfManifests": 2, - "totalDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" - }, - "totalExtractDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" - }, - "totalFetchFileDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + } }, - "totalLoadDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" - }, - "totalParseDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" - } + "numberOfFiles": 4, + "numberOfManifests": 2 } }, "severity": "note", diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected index 1c46a0a9f99c..3c4fc9bf4316 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected @@ -1,28 +1,30 @@ { "attributes": { "summary": { - "numberOfFiles": 4, - "numberOfManifests": 1, - "totalDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" - }, - "totalExtractDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" - }, - "totalFetchFileDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + } }, - "totalLoadDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" - }, - "totalParseDuration": { - "ms": "REDACTED", - "pretty": "REDACTED" - } + "numberOfFiles": 4, + "numberOfManifests": 1 } }, "severity": "note", From 5251dc2058cb684b617cb0ba72f6531b133f3976 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 27 Nov 2024 10:20:31 +0100 Subject: [PATCH 05/10] Rust: use `check_diagnostics` improvements --- rust/ql/integration-tests/conftest.py | 10 +- .../hello-project/diagnostics.expected | 123 ++++++++++++++---- .../diagnostics.cargo.expected | 72 ++++++++++ .../diagnostics.rust-project.expected | 67 ++++++++++ 4 files changed, 246 insertions(+), 26 deletions(-) diff --git a/rust/ql/integration-tests/conftest.py b/rust/ql/integration-tests/conftest.py index 895650183c88..cbace615805a 100644 --- a/rust/ql/integration-tests/conftest.py +++ b/rust/ql/integration-tests/conftest.py @@ -17,10 +17,12 @@ def manifests(cwd): @pytest.fixture def rust_check_diagnostics(check_diagnostics): check_diagnostics.replacements += [ - (r'"ms"\s*:\s*[0-9]+', '"ms": "__REDACTED__"'), - (r'"pretty"\s*:\s*"[^"]*"', '"pretty": "__REDACTED__"'), + ("Cargo.toml|rust-project.json", ""), ] - check_diagnostics.skip += [ - "attributes.steps", # the order of the steps is not stable + check_diagnostics.redact += [ + "attributes.summary.durations.*.ms", + "attributes.summary.durations.*.pretty", + "attributes.steps.ms", ] + check_diagnostics.sort = True # the order of the steps is not stable return check_diagnostics diff --git a/rust/ql/integration-tests/hello-project/diagnostics.expected b/rust/ql/integration-tests/hello-project/diagnostics.expected index c19e46a88cda..5f6821ffb104 100644 --- a/rust/ql/integration-tests/hello-project/diagnostics.expected +++ b/rust/ql/integration-tests/hello-project/diagnostics.expected @@ -1,28 +1,107 @@ { "attributes": { - "summary": { - "numberOfFiles": 4, - "numberOfManifests": 1, - "totalDuration": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "totalExtractDuration": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "totalFetchFileDuration": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "totalLoadDuration": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "totalParseDuration": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" + "steps": [ + { + "action": "extract", + "file": "/src/directory_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/src/directory_module/nested_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/src/directory_module/not_loaded.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/src/file_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/src/main.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadManifest", + "file": "/", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/src/directory_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/src/directory_module/nested_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/src/file_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/src/main.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/src/directory_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/src/directory_module/nested_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/src/directory_module/not_loaded.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/src/file_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/src/main.rs", + "ms": "__REDACTED__" } + ], + "summary": { + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + } + }, + "numberOfFiles": 5, + "numberOfManifests": 1 } }, "severity": "note", diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected index 052fe3e78393..7d4846230877 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected @@ -1,5 +1,77 @@ { "attributes": { + "steps": [ + { + "action": "extract", + "file": "/exe/src/a_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/exe/src/main.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/lib/src/a_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/lib/src/lib.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadManifest", + "file": "/exe/", + "ms": "__REDACTED__" + }, + { + "action": "loadManifest", + "file": "/lib/", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/exe/src/a_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/exe/src/main.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/lib/src/a_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/lib/src/lib.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/exe/src/a_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/exe/src/main.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/lib/src/a_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/lib/src/lib.rs", + "ms": "__REDACTED__" + } + ], "summary": { "durations": { "extract": { diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected index 3c4fc9bf4316..8ab17e9fe735 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected @@ -1,5 +1,72 @@ { "attributes": { + "steps": [ + { + "action": "extract", + "file": "/exe/src/a_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/exe/src/main.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/lib/src/a_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "extract", + "file": "/lib/src/lib.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadManifest", + "file": "/", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/exe/src/a_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/exe/src/main.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/lib/src/a_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "loadSource", + "file": "/lib/src/lib.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/exe/src/a_module.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/exe/src/main.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/lib/src/a_module/mod.rs", + "ms": "__REDACTED__" + }, + { + "action": "parse", + "file": "/lib/src/lib.rs", + "ms": "__REDACTED__" + } + ], "summary": { "durations": { "extract": { From 4e7115538bf8f63788c19694c1ea0b430b4fbe17 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 27 Nov 2024 14:55:46 +0100 Subject: [PATCH 06/10] Rust: move steps breakdown from JSON diagnostics to the DB --- rust/extractor/src/diagnostics.rs | 23 ++-- rust/extractor/src/main.rs | 24 +++- rust/extractor/src/trap.rs | 4 +- rust/prefix.dbscheme | 7 + rust/ql/integration-tests/conftest.py | 9 +- .../hello-project/diagnostics.expected | 125 ++++-------------- .../hello-project/steps.cargo.expected | 15 +++ .../integration-tests/hello-project/steps.ql | 4 + .../hello-project/steps.rust-project.expected | 15 +++ .../hello-project/test_project.py | 5 + .../diagnostics.cargo.expected | 120 ++++------------- .../diagnostics.rust-project.expected | 115 ++++------------ .../hello-workspace/steps.cargo.expected | 14 ++ .../hello-workspace/steps.ql | 4 + .../steps.rust-project.expected | 13 ++ .../hello-workspace/test_workspace.py | 6 +- .../codeql/rust/internal/ExtractorStep.qll | 46 +++++++ rust/ql/lib/rust.dbscheme | 7 + 18 files changed, 236 insertions(+), 320 deletions(-) create mode 100644 rust/ql/integration-tests/hello-project/steps.cargo.expected create mode 100644 rust/ql/integration-tests/hello-project/steps.ql create mode 100644 rust/ql/integration-tests/hello-project/steps.rust-project.expected create mode 100644 rust/ql/integration-tests/hello-workspace/steps.cargo.expected create mode 100644 rust/ql/integration-tests/hello-workspace/steps.ql create mode 100644 rust/ql/integration-tests/hello-workspace/steps.rust-project.expected create mode 100644 rust/ql/lib/codeql/rust/internal/ExtractorStep.qll diff --git a/rust/extractor/src/diagnostics.rs b/rust/extractor/src/diagnostics.rs index d4b60211bbfe..92743a923d4a 100644 --- a/rust/extractor/src/diagnostics.rs +++ b/rust/extractor/src/diagnostics.rs @@ -76,7 +76,7 @@ pub struct Diagnostics { #[derive(Default, Debug, Clone, Copy, Serialize, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] -enum ExtractionStepKind { +pub enum ExtractionStepKind { #[default] LoadManifest, LoadSource, @@ -87,9 +87,9 @@ enum ExtractionStepKind { #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct ExtractionStep { - action: ExtractionStepKind, - file: PathBuf, - ms: u128, + pub action: ExtractionStepKind, + pub file: PathBuf, + pub ms: u128, } impl ExtractionStep { @@ -186,14 +186,7 @@ struct ExtractionSummary { durations: DurationsSummary, } -#[derive(Debug, Default, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -struct ExtractionAttributes { - steps: Vec, - summary: ExtractionSummary, -} - -type ExtractionDiagnostics = Diagnostics; +type ExtractionDiagnostics = Diagnostics; fn summary(start: Instant, steps: &[ExtractionStep]) -> ExtractionSummary { let mut number_of_manifests = 0; @@ -229,9 +222,9 @@ fn summary(start: Instant, steps: &[ExtractionStep]) -> ExtractionSummary { pub fn emit_extraction_diagnostics( start: Instant, config: &Config, - steps: Vec, + steps: &[ExtractionStep], ) -> anyhow::Result<()> { - let summary = summary(start, &steps); + let summary = summary(start, steps); let diagnostics = ExtractionDiagnostics { source: Source { id: "rust/extractor/telemetry".to_owned(), @@ -243,7 +236,7 @@ pub fn emit_extraction_diagnostics( ..Default::default() }, timestamp: Utc::now(), - attributes: ExtractionAttributes { steps, summary }, + attributes: summary, ..Default::default() }; diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index 19239cf9dd48..a3258f16d096 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -2,7 +2,7 @@ use crate::diagnostics::{emit_extraction_diagnostics, ExtractionStep}; use crate::rust_analyzer::path_to_file_id; use anyhow::Context; use archive::Archiver; -use log::info; +use log::{info, warn}; use ra_ap_hir::Semantics; use ra_ap_ide_db::line_index::{LineCol, LineIndex}; use ra_ap_ide_db::RootDatabase; @@ -140,7 +140,27 @@ impl<'a> Extractor<'a> { start: Instant, cfg: &config::Config, ) -> anyhow::Result<()> { - emit_extraction_diagnostics(start, cfg, self.steps) + emit_extraction_diagnostics(start, cfg, &self.steps)?; + let mut trap = self.traps.create("diagnostics", "extraction"); + for step in self.steps { + let file_label = trap.emit_file(&step.file); + let step_label = trap.writer.fresh_id(); + let ms = usize::try_from(step.ms).unwrap_or_else(|_e| { + warn!("extraction step duration overflowed ({step:?})"); + i32::MAX as usize + }); + trap.writer.add_tuple( + "extractor_steps", + vec![ + step_label.into(), + format!("{:?}", step.action).into(), + file_label.into(), + ms.into(), + ], + ); + } + trap.commit()?; + Ok(()) } } diff --git a/rust/extractor/src/trap.rs b/rust/extractor/src/trap.rs index 58f56fb2b1e5..c993fa046a7c 100644 --- a/rust/extractor/src/trap.rs +++ b/rust/extractor/src/trap.rs @@ -243,8 +243,8 @@ impl TrapFileProvider { }) } - pub fn create(&self, category: &str, key: &Path) -> TrapFile { - let path = file_paths::path_for(&self.trap_dir.join(category), key, "trap"); + pub fn create(&self, category: &str, key: impl AsRef) -> TrapFile { + let path = file_paths::path_for(&self.trap_dir.join(category), key.as_ref(), "trap"); debug!("creating trap file {}", path.display()); let mut writer = trap::Writer::new(); extractor::populate_empty_location(&mut writer); diff --git a/rust/prefix.dbscheme b/rust/prefix.dbscheme index 4810e11069d4..5ccbd3438d13 100644 --- a/rust/prefix.dbscheme +++ b/rust/prefix.dbscheme @@ -3,3 +3,10 @@ locatable_locations( int id: @locatable ref, int location: @location_default ref ); + +extractor_steps( + unique int id: @extractor_step, + string action: string ref, + int file: @file ref, + int duration_ms: int ref +) diff --git a/rust/ql/integration-tests/conftest.py b/rust/ql/integration-tests/conftest.py index cbace615805a..79c899c97fc7 100644 --- a/rust/ql/integration-tests/conftest.py +++ b/rust/ql/integration-tests/conftest.py @@ -16,13 +16,8 @@ def manifests(cwd): @pytest.fixture def rust_check_diagnostics(check_diagnostics): - check_diagnostics.replacements += [ - ("Cargo.toml|rust-project.json", ""), - ] check_diagnostics.redact += [ - "attributes.summary.durations.*.ms", - "attributes.summary.durations.*.pretty", - "attributes.steps.ms", + "attributes.durations.*.ms", + "attributes.durations.*.pretty", ] - check_diagnostics.sort = True # the order of the steps is not stable return check_diagnostics diff --git a/rust/ql/integration-tests/hello-project/diagnostics.expected b/rust/ql/integration-tests/hello-project/diagnostics.expected index 5f6821ffb104..f938b5b8ab5b 100644 --- a/rust/ql/integration-tests/hello-project/diagnostics.expected +++ b/rust/ql/integration-tests/hello-project/diagnostics.expected @@ -1,108 +1,29 @@ { "attributes": { - "steps": [ - { - "action": "extract", - "file": "/src/directory_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/src/directory_module/nested_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/src/directory_module/not_loaded.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/src/file_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/src/main.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadManifest", - "file": "/", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/src/directory_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/src/directory_module/nested_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/src/file_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/src/main.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/src/directory_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/src/directory_module/nested_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/src/directory_module/not_loaded.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/src/file_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/src/main.rs", - "ms": "__REDACTED__" + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" } - ], - "summary": { - "durations": { - "extract": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "loadManifest": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "loadSource": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "parse": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "total": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - } - }, - "numberOfFiles": 5, - "numberOfManifests": 1 - } + }, + "numberOfFiles": 5, + "numberOfManifests": 1 }, "severity": "note", "source": { diff --git a/rust/ql/integration-tests/hello-project/steps.cargo.expected b/rust/ql/integration-tests/hello-project/steps.cargo.expected new file mode 100644 index 000000000000..61987aa564bb --- /dev/null +++ b/rust/ql/integration-tests/hello-project/steps.cargo.expected @@ -0,0 +1,15 @@ +| Cargo.toml:0:0:0:0 | LoadManifest(Cargo.toml) | +| src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) | +| src/directory_module/mod.rs:0:0:0:0 | LoadSource(src/directory_module/mod.rs) | +| src/directory_module/mod.rs:0:0:0:0 | Parse(src/directory_module/mod.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | Extract(src/directory_module/nested_module.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | LoadSource(src/directory_module/nested_module.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | Parse(src/directory_module/nested_module.rs) | +| src/directory_module/not_loaded.rs:0:0:0:0 | Extract(src/directory_module/not_loaded.rs) | +| src/directory_module/not_loaded.rs:0:0:0:0 | Parse(src/directory_module/not_loaded.rs) | +| src/file_module.rs:0:0:0:0 | Extract(src/file_module.rs) | +| src/file_module.rs:0:0:0:0 | LoadSource(src/file_module.rs) | +| src/file_module.rs:0:0:0:0 | Parse(src/file_module.rs) | +| src/main.rs:0:0:0:0 | Extract(src/main.rs) | +| src/main.rs:0:0:0:0 | LoadSource(src/main.rs) | +| src/main.rs:0:0:0:0 | Parse(src/main.rs) | diff --git a/rust/ql/integration-tests/hello-project/steps.ql b/rust/ql/integration-tests/hello-project/steps.ql new file mode 100644 index 000000000000..eb8f9a05adb7 --- /dev/null +++ b/rust/ql/integration-tests/hello-project/steps.ql @@ -0,0 +1,4 @@ +import codeql.rust.internal.ExtractorStep + +from ExtractorStep step +select step diff --git a/rust/ql/integration-tests/hello-project/steps.rust-project.expected b/rust/ql/integration-tests/hello-project/steps.rust-project.expected new file mode 100644 index 000000000000..d1d2e9ddee3f --- /dev/null +++ b/rust/ql/integration-tests/hello-project/steps.rust-project.expected @@ -0,0 +1,15 @@ +| rust-project.json:0:0:0:0 | LoadManifest(rust-project.json) | +| src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) | +| src/directory_module/mod.rs:0:0:0:0 | LoadSource(src/directory_module/mod.rs) | +| src/directory_module/mod.rs:0:0:0:0 | Parse(src/directory_module/mod.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | Extract(src/directory_module/nested_module.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | LoadSource(src/directory_module/nested_module.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | Parse(src/directory_module/nested_module.rs) | +| src/directory_module/not_loaded.rs:0:0:0:0 | Extract(src/directory_module/not_loaded.rs) | +| src/directory_module/not_loaded.rs:0:0:0:0 | Parse(src/directory_module/not_loaded.rs) | +| src/file_module.rs:0:0:0:0 | Extract(src/file_module.rs) | +| src/file_module.rs:0:0:0:0 | LoadSource(src/file_module.rs) | +| src/file_module.rs:0:0:0:0 | Parse(src/file_module.rs) | +| src/main.rs:0:0:0:0 | Extract(src/main.rs) | +| src/main.rs:0:0:0:0 | LoadSource(src/main.rs) | +| src/main.rs:0:0:0:0 | Parse(src/main.rs) | diff --git a/rust/ql/integration-tests/hello-project/test_project.py b/rust/ql/integration-tests/hello-project/test_project.py index 2cbac0ffdb8a..64988bef7a9c 100644 --- a/rust/ql/integration-tests/hello-project/test_project.py +++ b/rust/ql/integration-tests/hello-project/test_project.py @@ -1,7 +1,12 @@ +import pytest + + +@pytest.mark.ql_test("steps.ql", expected=".cargo.expected") def test_cargo(codeql, rust, manifests, check_source_archive, rust_check_diagnostics): manifests.select("Cargo.toml") codeql.database.create() +@pytest.mark.ql_test("steps.ql", expected=".rust-project.expected") def test_rust_project(codeql, rust, manifests, check_source_archive, rust_check_diagnostics): manifests.select("rust-project.json") codeql.database.create() diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected index 7d4846230877..7d44256db5a4 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected @@ -1,103 +1,29 @@ { "attributes": { - "steps": [ - { - "action": "extract", - "file": "/exe/src/a_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/exe/src/main.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/lib/src/a_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/lib/src/lib.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadManifest", - "file": "/exe/", - "ms": "__REDACTED__" - }, - { - "action": "loadManifest", - "file": "/lib/", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/exe/src/a_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/exe/src/main.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/lib/src/a_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/lib/src/lib.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/exe/src/a_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/exe/src/main.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/lib/src/a_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/lib/src/lib.rs", - "ms": "__REDACTED__" + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" } - ], - "summary": { - "durations": { - "extract": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "loadManifest": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "loadSource": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "parse": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "total": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - } - }, - "numberOfFiles": 4, - "numberOfManifests": 2 - } + }, + "numberOfFiles": 4, + "numberOfManifests": 2 }, "severity": "note", "source": { diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected index 8ab17e9fe735..4c5dbc75d84b 100644 --- a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected @@ -1,98 +1,29 @@ { "attributes": { - "steps": [ - { - "action": "extract", - "file": "/exe/src/a_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/exe/src/main.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/lib/src/a_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "extract", - "file": "/lib/src/lib.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadManifest", - "file": "/", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/exe/src/a_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/exe/src/main.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/lib/src/a_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "loadSource", - "file": "/lib/src/lib.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/exe/src/a_module.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/exe/src/main.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/lib/src/a_module/mod.rs", - "ms": "__REDACTED__" - }, - { - "action": "parse", - "file": "/lib/src/lib.rs", - "ms": "__REDACTED__" + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" } - ], - "summary": { - "durations": { - "extract": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "loadManifest": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "loadSource": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "parse": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - }, - "total": { - "ms": "__REDACTED__", - "pretty": "__REDACTED__" - } - }, - "numberOfFiles": 4, - "numberOfManifests": 1 - } + }, + "numberOfFiles": 4, + "numberOfManifests": 1 }, "severity": "note", "source": { diff --git a/rust/ql/integration-tests/hello-workspace/steps.cargo.expected b/rust/ql/integration-tests/hello-workspace/steps.cargo.expected new file mode 100644 index 000000000000..138bd77b690d --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/steps.cargo.expected @@ -0,0 +1,14 @@ +| exe/Cargo.toml:0:0:0:0 | LoadManifest(exe/Cargo.toml) | +| exe/src/a_module.rs:0:0:0:0 | Extract(exe/src/a_module.rs) | +| exe/src/a_module.rs:0:0:0:0 | LoadSource(exe/src/a_module.rs) | +| exe/src/a_module.rs:0:0:0:0 | Parse(exe/src/a_module.rs) | +| exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) | +| exe/src/main.rs:0:0:0:0 | LoadSource(exe/src/main.rs) | +| exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) | +| lib/Cargo.toml:0:0:0:0 | LoadManifest(lib/Cargo.toml) | +| lib/src/a_module/mod.rs:0:0:0:0 | Extract(lib/src/a_module/mod.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | LoadSource(lib/src/a_module/mod.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | Parse(lib/src/a_module/mod.rs) | +| lib/src/lib.rs:0:0:0:0 | Extract(lib/src/lib.rs) | +| lib/src/lib.rs:0:0:0:0 | LoadSource(lib/src/lib.rs) | +| lib/src/lib.rs:0:0:0:0 | Parse(lib/src/lib.rs) | diff --git a/rust/ql/integration-tests/hello-workspace/steps.ql b/rust/ql/integration-tests/hello-workspace/steps.ql new file mode 100644 index 000000000000..eb8f9a05adb7 --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/steps.ql @@ -0,0 +1,4 @@ +import codeql.rust.internal.ExtractorStep + +from ExtractorStep step +select step diff --git a/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected b/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected new file mode 100644 index 000000000000..09620925ce96 --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected @@ -0,0 +1,13 @@ +| exe/src/a_module.rs:0:0:0:0 | Extract(exe/src/a_module.rs) | +| exe/src/a_module.rs:0:0:0:0 | LoadSource(exe/src/a_module.rs) | +| exe/src/a_module.rs:0:0:0:0 | Parse(exe/src/a_module.rs) | +| exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) | +| exe/src/main.rs:0:0:0:0 | LoadSource(exe/src/main.rs) | +| exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | Extract(lib/src/a_module/mod.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | LoadSource(lib/src/a_module/mod.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | Parse(lib/src/a_module/mod.rs) | +| lib/src/lib.rs:0:0:0:0 | Extract(lib/src/lib.rs) | +| lib/src/lib.rs:0:0:0:0 | LoadSource(lib/src/lib.rs) | +| lib/src/lib.rs:0:0:0:0 | Parse(lib/src/lib.rs) | +| rust-project.json:0:0:0:0 | LoadManifest(rust-project.json) | diff --git a/rust/ql/integration-tests/hello-workspace/test_workspace.py b/rust/ql/integration-tests/hello-workspace/test_workspace.py index 5c95031466f8..3dd49991aaac 100644 --- a/rust/ql/integration-tests/hello-workspace/test_workspace.py +++ b/rust/ql/integration-tests/hello-workspace/test_workspace.py @@ -1,13 +1,13 @@ import pytest -# currently the DB-check fails on actions because of loading files multiple times and assiging multiple locations -# see https://github.com/github/codeql-team/issues/3365 -@pytest.mark.ql_test("DB-CHECK", xfail="maybe") + +@pytest.mark.ql_test("steps.ql", expected=".cargo.expected") def test_cargo(codeql, rust, manifests, check_source_archive, rust_check_diagnostics): rust_check_diagnostics.expected_suffix = ".cargo.expected" manifests.select("Cargo.toml") codeql.database.create() +@pytest.mark.ql_test("steps.ql", expected=".rust-project.expected") def test_rust_project(codeql, rust, manifests, check_source_archive, rust_check_diagnostics): rust_check_diagnostics.expected_suffix = ".rust-project.expected" manifests.select("rust-project.json") diff --git a/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll b/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll new file mode 100644 index 000000000000..55505be1d591 --- /dev/null +++ b/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll @@ -0,0 +1,46 @@ +import codeql.files.FileSystem + +/** + * An extractor step, usable for debugging and diagnostics reasons. + * + * INTERNAL: Do not use. + */ +class ExtractorStep extends @extractor_step { + /** + * The string representation of this extractor step. + */ + string toString() { + exists(File file, string action | + extractor_steps(this, action, file, _) and + result = action + "(" + file.getAbsolutePath() + ")" + ) + } + + /** + * The action this extractor step carried out. + */ + string getAction() { extractor_steps(this, result, _, _) } + + /** + * The file the extractor step was carried out on. + */ + File getFile() { extractor_steps(this, _, result, _) } + + /** + * The duration of the extractor step in milliseconds. + */ + int getDurationMs() { extractor_steps(this, _, _, result) } + + /** + * Provides location information for this step. + */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + this.getFile().getAbsolutePath() = filepath and + startline = 0 and + startcolumn = 0 and + endline = 0 and + endcolumn = 0 + } +} diff --git a/rust/ql/lib/rust.dbscheme b/rust/ql/lib/rust.dbscheme index 46085a55abfe..26a22063d6df 100644 --- a/rust/ql/lib/rust.dbscheme +++ b/rust/ql/lib/rust.dbscheme @@ -116,6 +116,13 @@ locatable_locations( int location: @location_default ref ); +extractor_steps( + unique int id: @extractor_step, + string action: string ref, + int file: @file ref, + int duration_ms: int ref +) + // from schema From 50c917d2eb9fe8f53464b7aea841db3bbc2e55a1 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 28 Nov 2024 12:02:57 +0100 Subject: [PATCH 07/10] Rust: restrict extracted files queries --- rust/ql/lib/codeql/files/FileSystem.qll | 16 ++++++++++++++-- .../lib/codeql/rust/internal/ExtractorStep.qll | 8 ++++---- .../ql/src/queries/diagnostics/ExtractedFiles.ql | 2 +- .../src/queries/diagnostics/ExtractionErrors.ql | 2 +- rust/ql/src/queries/summary/SummaryStats.ql | 7 +++++-- rust/ql/test/extractor-tests/File/File.expected | 7 ++++--- rust/ql/test/extractor-tests/File/File.ql | 8 +++++--- .../diagnostics/ExtractedFiles.expected | 1 - .../diagnostics/LinesOfUserCodeInFiles.expected | 1 + .../diagnostics/SummaryStats.expected | 4 ++-- 10 files changed, 37 insertions(+), 19 deletions(-) diff --git a/rust/ql/lib/codeql/files/FileSystem.qll b/rust/ql/lib/codeql/files/FileSystem.qll index 7136dcc3b5a2..c83680f7ec69 100644 --- a/rust/ql/lib/codeql/files/FileSystem.qll +++ b/rust/ql/lib/codeql/files/FileSystem.qll @@ -6,6 +6,7 @@ private import codeql.rust.elements.SourceFile private import codeql.rust.elements.AstNode private import codeql.rust.elements.Comment private import codeql.rust.Diagnostics +private import codeql.rust.internal.ExtractorStep private module Input implements InputSig { abstract class ContainerBase extends @container { @@ -36,7 +37,9 @@ class Folder = Impl::Folder; /** A file. */ class File extends Container, Impl::File { /** Holds if this file was extracted from ordinary source code. */ - predicate fromSource() { any() } + predicate fromSource() { + exists(ExtractorStep s | s.getAction() = "Extract" and s.getFile() = this) + } /** * Gets the number of lines containing code in this file. This value @@ -58,11 +61,20 @@ class File extends Container, Impl::File { } } +/** + * A source file that was extracted. + * + * TODO: rename `SourceFile` from the generated AST to give that name to this class. + */ +class ExtractedFile extends File { + ExtractedFile() { this.fromSource() } +} + /** * A successfully extracted file, that is, a file that was extracted and * contains no extraction errors or warnings. */ -class SuccessfullyExtractedFile extends File { +class SuccessfullyExtractedFile extends ExtractedFile { SuccessfullyExtractedFile() { not exists(Diagnostic d | d.getLocation().getFile() = this and diff --git a/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll b/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll index 55505be1d591..50e6ebafebfe 100644 --- a/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll +++ b/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll @@ -7,7 +7,7 @@ import codeql.files.FileSystem */ class ExtractorStep extends @extractor_step { /** - * The string representation of this extractor step. + * Gets the string representation of this extractor step. */ string toString() { exists(File file, string action | @@ -17,17 +17,17 @@ class ExtractorStep extends @extractor_step { } /** - * The action this extractor step carried out. + * Gets the action this extractor step carried out. */ string getAction() { extractor_steps(this, result, _, _) } /** - * The file the extractor step was carried out on. + * Gets the file the extractor step was carried out on. */ File getFile() { extractor_steps(this, _, result, _) } /** - * The duration of the extractor step in milliseconds. + * Gets the duration of the extractor step in milliseconds. */ int getDurationMs() { extractor_steps(this, _, _, result) } diff --git a/rust/ql/src/queries/diagnostics/ExtractedFiles.ql b/rust/ql/src/queries/diagnostics/ExtractedFiles.ql index ce5aa699e671..112fb2949dc2 100644 --- a/rust/ql/src/queries/diagnostics/ExtractedFiles.ql +++ b/rust/ql/src/queries/diagnostics/ExtractedFiles.ql @@ -8,6 +8,6 @@ import rust -from File f +from ExtractedFile f where exists(f.getRelativePath()) select f, "File successfully extracted." diff --git a/rust/ql/src/queries/diagnostics/ExtractionErrors.ql b/rust/ql/src/queries/diagnostics/ExtractionErrors.ql index 68be66f8ca9b..bd5ec3426fb6 100644 --- a/rust/ql/src/queries/diagnostics/ExtractionErrors.ql +++ b/rust/ql/src/queries/diagnostics/ExtractionErrors.ql @@ -11,7 +11,7 @@ import codeql.files.FileSystem /** Gets the SARIF severity to associate with an error. */ int getSeverity() { result = 2 } -from ExtractionError error, File f +from ExtractionError error, ExtractedFile f where f = error.getLocation().getFile() and exists(f.getRelativePath()) diff --git a/rust/ql/src/queries/summary/SummaryStats.ql b/rust/ql/src/queries/summary/SummaryStats.ql index 09ee83fc5e64..ffe7cbf1a8fa 100644 --- a/rust/ql/src/queries/summary/SummaryStats.ql +++ b/rust/ql/src/queries/summary/SummaryStats.ql @@ -21,10 +21,13 @@ where or key = "Extraction warnings" and value = count(ExtractionWarning w) or - key = "Files extracted - total" and value = count(File f | exists(f.getRelativePath())) + key = "Files extracted - total" and value = count(ExtractedFile f | exists(f.getRelativePath())) or key = "Files extracted - with errors" and - value = count(File f | exists(f.getRelativePath()) and not f instanceof SuccessfullyExtractedFile) + value = + count(ExtractedFile f | + exists(f.getRelativePath()) and not f instanceof SuccessfullyExtractedFile + ) or key = "Files extracted - without errors" and value = count(SuccessfullyExtractedFile f | exists(f.getRelativePath())) diff --git a/rust/ql/test/extractor-tests/File/File.expected b/rust/ql/test/extractor-tests/File/File.expected index 924ed370b35b..77fee3c3c428 100644 --- a/rust/ql/test/extractor-tests/File/File.expected +++ b/rust/ql/test/extractor-tests/File/File.expected @@ -1,3 +1,4 @@ -| a_file.rs:0:0:0:0 | a_file.rs | -| another_file.rs:0:0:0:0 | another_file.rs | -| lib.rs:0:0:0:0 | lib.rs | +| Cargo.toml:0:0:0:0 | Cargo.toml | fromSource: no | +| a_file.rs:0:0:0:0 | a_file.rs | fromSource: yes | +| another_file.rs:0:0:0:0 | another_file.rs | fromSource: yes | +| lib.rs:0:0:0:0 | lib.rs | fromSource: yes | diff --git a/rust/ql/test/extractor-tests/File/File.ql b/rust/ql/test/extractor-tests/File/File.ql index fcb2b274e781..316099193d15 100644 --- a/rust/ql/test/extractor-tests/File/File.ql +++ b/rust/ql/test/extractor-tests/File/File.ql @@ -1,5 +1,7 @@ import rust -from File f -where exists(f.getRelativePath()) -select f +from File f, string fromSource +where + exists(f.getRelativePath()) and + if f.fromSource() then fromSource = "fromSource: yes" else fromSource = "fromSource: no" +select f, fromSource diff --git a/rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected b/rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected index bc8dc8cccf15..c5ebb7072304 100644 --- a/rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected +++ b/rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected @@ -5,4 +5,3 @@ | main.rs:0:0:0:0 | main.rs | File successfully extracted. | | my_macro.rs:0:0:0:0 | my_macro.rs | File successfully extracted. | | my_struct.rs:0:0:0:0 | my_struct.rs | File successfully extracted. | -| options.yml:0:0:0:0 | options.yml | File successfully extracted. | diff --git a/rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected b/rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected index 0a58a05feed2..f53c931e56ee 100644 --- a/rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected +++ b/rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected @@ -5,4 +5,5 @@ | lib.rs:0:0:0:0 | lib.rs | 5 | | does_not_compile.rs:0:0:0:0 | does_not_compile.rs | 3 | | error.rs:0:0:0:0 | error.rs | 3 | +| Cargo.toml:0:0:0:0 | Cargo.toml | 0 | | options.yml:0:0:0:0 | options.yml | 0 | diff --git a/rust/ql/test/query-tests/diagnostics/SummaryStats.expected b/rust/ql/test/query-tests/diagnostics/SummaryStats.expected index 74e1e461c6fe..16e895cdeef3 100644 --- a/rust/ql/test/query-tests/diagnostics/SummaryStats.expected +++ b/rust/ql/test/query-tests/diagnostics/SummaryStats.expected @@ -2,9 +2,9 @@ | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 7 | -| Files extracted - total | 8 | +| Files extracted - total | 7 | | Files extracted - with errors | 3 | -| Files extracted - without errors | 5 | +| Files extracted - without errors | 4 | | Inconsistencies - AST | 0 | | Inconsistencies - CFG | 0 | | Inconsistencies - data flow | 0 | From 7e0e5a3f4efd2468be1108e63c6ef4a5662ef249 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 29 Nov 2024 12:01:50 +0100 Subject: [PATCH 08/10] Rust: move `rust_sysroot_src` to its own session fixture --- rust/ql/integration-tests/conftest.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/rust/ql/integration-tests/conftest.py b/rust/ql/integration-tests/conftest.py index 11eefe7e2650..a1fbcf4e18d2 100644 --- a/rust/ql/integration-tests/conftest.py +++ b/rust/ql/integration-tests/conftest.py @@ -4,18 +4,24 @@ import pathlib +@pytest.fixture def cargo(cwd): assert (cwd / "Cargo.toml").exists() (cwd / "rust-project.json").unlink(missing_ok=True) +@pytest.fixture(scope="session") +def rust_sysroot_src() -> str: + rust_sysroot = pathlib.Path(commands.run("rustc --print sysroot", _capture=True)) + ret = rust_sysroot.joinpath("lib", "rustlib", "src", "rust", "library") + assert ret.exists() + return str(ret) @pytest.fixture -def rust_project(cwd): +def rust_project(cwd, rust_sysroot_src): project_file = cwd / "rust-project.json" assert project_file.exists() - rust_sysroot = pathlib.Path(commands.run("rustc --print sysroot", _capture=True)) project = json.loads(project_file.read_text()) - project["sysroot_src"] = str(rust_sysroot.joinpath("lib", "rustlib", "src", "rust", "library")) + project["sysroot_src"] = rust_sysroot_src project_file.write_text(json.dumps(project, indent=4)) (cwd / "Cargo.toml").unlink(missing_ok=True) From b57a37479b25648a0cdd3925e10a9ed85f779fc7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 2 Dec 2024 15:15:46 +0100 Subject: [PATCH 09/10] Rust: make `File` usable in `codegen` --- misc/codegen/generators/dbschemegen.py | 3 +- misc/codegen/generators/qlgen.py | 35 ++++++++++---- misc/codegen/generators/rustgen.py | 16 ++++--- misc/codegen/generators/rusttestgen.py | 2 + misc/codegen/lib/ql.py | 6 +-- misc/codegen/lib/schema.py | 28 ++++++++--- misc/codegen/lib/schemadefs.py | 7 +-- misc/codegen/loaders/schemaloader.py | 6 ++- misc/codegen/templates/ql_class.mustache | 2 +- misc/codegen/templates/ql_db.mustache | 4 ++ misc/codegen/test/test_ql.py | 15 ------ misc/codegen/test/test_qlgen.py | 4 +- rust/extractor/src/generated/.generated.list | 2 +- rust/extractor/src/generated/top.rs | 40 ++++++++++++++++ rust/extractor/src/main.rs | 21 ++++----- rust/extractor/src/translate/base.rs | 5 +- rust/extractor/src/trap.rs | 14 +++--- rust/prefix.dbscheme | 7 --- rust/ql/.generated.list | 13 ++++-- rust/ql/.gitattributes | 3 ++ .../integration-tests/hello-project/steps.ql | 2 +- .../hello-project/summary.expected | 2 +- .../hello-workspace/steps.ql | 2 +- ...ummary.expected => summary.cargo.expected} | 2 +- .../summary.rust-project.expected | 17 +++++++ .../hello-workspace/test_workspace.py | 2 + rust/ql/lib/codeql/files/FileSystem.qll | 2 +- rust/ql/lib/codeql/rust/elements.qll | 1 + .../rust/elements/internal/ExtractorStep.qll | 13 ++++++ .../internal/ExtractorStepConstructor.qll | 14 ++++++ .../elements/internal/ExtractorStepImpl.qll | 32 +++++++++++++ .../internal/generated/ExtractorStep.qll | 45 ++++++++++++++++++ .../internal/generated/ParentChild.qll | 18 ++++++++ .../rust/elements/internal/generated/Raw.qll | 24 ++++++++++ .../elements/internal/generated/Synth.qll | 20 ++++++++ .../internal/generated/SynthConstructors.qll | 1 + .../codeql/rust/internal/ExtractorStep.qll | 46 ------------------- rust/ql/lib/rust.dbscheme | 17 +++---- rust/schema/prelude.py | 9 ++++ swift/ql/.generated.list | 2 +- 40 files changed, 363 insertions(+), 141 deletions(-) rename rust/ql/integration-tests/hello-workspace/{summary.expected => summary.cargo.expected} (94%) create mode 100644 rust/ql/integration-tests/hello-workspace/summary.rust-project.expected create mode 100644 rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll create mode 100644 rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll create mode 100644 rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll create mode 100644 rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll delete mode 100644 rust/ql/lib/codeql/rust/internal/ExtractorStep.qll diff --git a/misc/codegen/generators/dbschemegen.py b/misc/codegen/generators/dbschemegen.py index 2c3cd5598d55..e2cc4220dc71 100755 --- a/misc/codegen/generators/dbschemegen.py +++ b/misc/codegen/generators/dbschemegen.py @@ -110,7 +110,8 @@ def cls_to_dbscheme(cls: schema.Class, lookup: typing.Dict[str, schema.Class], a def get_declarations(data: schema.Schema): add_or_none_except = data.root_class.name if data.null else None - declarations = [d for cls in data.classes.values() for d in cls_to_dbscheme(cls, data.classes, add_or_none_except)] + declarations = [d for cls in data.classes.values() if not cls.imported for d in cls_to_dbscheme(cls, + data.classes, add_or_none_except)] if data.null: property_classes = { prop.type for cls in data.classes.values() for prop in cls.properties diff --git a/misc/codegen/generators/qlgen.py b/misc/codegen/generators/qlgen.py index e42c9d015522..feb2250bf3bb 100755 --- a/misc/codegen/generators/qlgen.py +++ b/misc/codegen/generators/qlgen.py @@ -104,8 +104,17 @@ def _get_doc(cls: schema.Class, prop: schema.Property, plural=None): return f"{prop_name} of this {class_name}" -def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.Class], +def _type_is_hideable(t: str, lookup: typing.Dict[str, schema.ClassBase]) -> bool: + if t in lookup: + match lookup[t]: + case schema.Class() as cls: + return "ql_hideable" in cls.pragmas + return False + + +def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.ClassBase], prev_child: str = "") -> ql.Property: + args = dict( type=prop.type if not prop.is_predicate else "predicate", qltest_skip="qltest_skip" in prop.pragmas, @@ -115,7 +124,8 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic is_unordered=prop.is_unordered, description=prop.description, synth=bool(cls.synth) or prop.synth, - type_is_hideable="ql_hideable" in lookup[prop.type].pragmas if prop.type in lookup else False, + type_is_hideable=_type_is_hideable(prop.type, lookup), + type_is_codegen_class=prop.type in lookup and not lookup[prop.type].imported, internal="ql_internal" in prop.pragmas, ) ql_name = prop.pragmas.get("ql_name", prop.name) @@ -154,7 +164,7 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic return ql.Property(**args) -def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]) -> ql.Class: +def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.ClassBase]) -> ql.Class: if "ql_name" in cls.pragmas: raise Error("ql_name is not supported yet for classes, only for properties") prev_child = "" @@ -391,14 +401,15 @@ def generate(opts, renderer): data = schemaloader.load_file(input) - classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items()} + classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items() if not cls.imported} if not classes: raise NoClasses root = next(iter(classes.values())) if root.has_children: raise RootElementHasChildren(root) - imports = {} + pre_imports = {n: cls.module for n, cls in data.classes.items() if cls.imported} + imports = dict(pre_imports) imports_impl = {} classes_used_by = {} cfg_classes = [] @@ -410,7 +421,7 @@ def generate(opts, renderer): force=opts.force) as renderer: db_classes = [cls for name, cls in classes.items() if not data.classes[name].synth] - renderer.render(ql.DbClasses(db_classes), out / "Raw.qll") + renderer.render(ql.DbClasses(classes=db_classes, imports=sorted(set(pre_imports.values()))), out / "Raw.qll") classes_by_dir_and_name = sorted(classes.values(), key=lambda cls: (cls.dir, cls.name)) for c in classes_by_dir_and_name: @@ -439,6 +450,8 @@ def generate(opts, renderer): renderer.render(cfg_classes_val, cfg_qll) for c in data.classes.values(): + if c.imported: + continue path = _get_path(c) path_impl = _get_path_impl(c) stub_file = stub_out / path_impl @@ -457,7 +470,7 @@ def generate(opts, renderer): renderer.render(class_public, class_public_file) # for example path/to/elements -> path/to/elements.qll - renderer.render(ql.ImportList([i for name, i in imports.items() if not classes[name].internal]), + renderer.render(ql.ImportList([i for name, i in imports.items() if name not in classes or not classes[name].internal]), include_file) elements_module = get_import(include_file, opts.root_dir) @@ -465,12 +478,15 @@ def generate(opts, renderer): renderer.render( ql.GetParentImplementation( classes=list(classes.values()), - imports=[elements_module] + [i for name, i in imports.items() if classes[name].internal], + imports=[elements_module] + [i for name, + i in imports.items() if name in classes and classes[name].internal], ), out / 'ParentChild.qll') if test_out: for c in data.classes.values(): + if c.imported: + continue if should_skip_qltest(c, data.classes): continue test_with_name = c.pragmas.get("qltest_test_with") @@ -500,7 +516,8 @@ def generate(opts, renderer): constructor_imports = [] synth_constructor_imports = [] stubs = {} - for cls in sorted(data.classes.values(), key=lambda cls: (cls.group, cls.name)): + for cls in sorted((cls for cls in data.classes.values() if not cls.imported), + key=lambda cls: (cls.group, cls.name)): synth_type = get_ql_synth_class(cls) if synth_type.is_final: final_synth_types.append(synth_type) diff --git a/misc/codegen/generators/rustgen.py b/misc/codegen/generators/rustgen.py index f4c977c0fdbe..b47e5cc4bd9d 100644 --- a/misc/codegen/generators/rustgen.py +++ b/misc/codegen/generators/rustgen.py @@ -49,7 +49,7 @@ def _get_field(cls: schema.Class, p: schema.Property) -> rust.Field: def _get_properties( - cls: schema.Class, lookup: dict[str, schema.Class], + cls: schema.Class, lookup: dict[str, schema.ClassBase], ) -> typing.Iterable[tuple[schema.Class, schema.Property]]: for b in cls.bases: yield from _get_properties(lookup[b], lookup) @@ -58,12 +58,14 @@ def _get_properties( def _get_ancestors( - cls: schema.Class, lookup: dict[str, schema.Class] + cls: schema.Class, lookup: dict[str, schema.ClassBase] ) -> typing.Iterable[schema.Class]: for b in cls.bases: base = lookup[b] - yield base - yield from _get_ancestors(base, lookup) + if not base.imported: + base = typing.cast(schema.Class, base) + yield base + yield from _get_ancestors(base, lookup) class Processor: @@ -71,7 +73,7 @@ def __init__(self, data: schema.Schema): self._classmap = data.classes def _get_class(self, name: str) -> rust.Class: - cls = self._classmap[name] + cls = typing.cast(schema.Class, self._classmap[name]) properties = [ (c, p) for c, p in _get_properties(cls, self._classmap) @@ -101,8 +103,10 @@ def _get_class(self, name: str) -> rust.Class: def get_classes(self): ret = {"": []} for k, cls in self._classmap.items(): - if not cls.synth: + if not cls.imported and not cls.synth: ret.setdefault(cls.group, []).append(self._get_class(cls.name)) + elif cls.imported: + ret[""].append(rust.Class(name=cls.name)) return ret diff --git a/misc/codegen/generators/rusttestgen.py b/misc/codegen/generators/rusttestgen.py index d360db27a658..e7a23fedacdc 100644 --- a/misc/codegen/generators/rusttestgen.py +++ b/misc/codegen/generators/rusttestgen.py @@ -56,6 +56,8 @@ def generate(opts, renderer): registry=opts.ql_test_output / ".generated_tests.list", force=opts.force) as renderer: for cls in schema.classes.values(): + if cls.imported: + continue if (qlgen.should_skip_qltest(cls, schema.classes) or "rust_skip_doc_test" in cls.pragmas): continue diff --git a/misc/codegen/lib/ql.py b/misc/codegen/lib/ql.py index 1182c7f5dc16..b9362a556ef1 100644 --- a/misc/codegen/lib/ql.py +++ b/misc/codegen/lib/ql.py @@ -44,6 +44,7 @@ class Property: doc_plural: Optional[str] = None synth: bool = False type_is_hideable: bool = False + type_is_codegen_class: bool = False internal: bool = False cfg: bool = False @@ -66,10 +67,6 @@ def indefinite_getter(self): article = "An" if self.singular[0] in "AEIO" else "A" return f"get{article}{self.singular}" - @property - def type_is_class(self): - return bool(self.type) and self.type[0].isupper() - @property def is_repeated(self): return bool(self.plural) @@ -191,6 +188,7 @@ class DbClasses: template: ClassVar = 'ql_db' classes: List[Class] = field(default_factory=list) + imports: List[str] = field(default_factory=list) @dataclass diff --git a/misc/codegen/lib/schema.py b/misc/codegen/lib/schema.py index 23f1aea2ba42..5178e61d3844 100644 --- a/misc/codegen/lib/schema.py +++ b/misc/codegen/lib/schema.py @@ -3,7 +3,7 @@ import typing from collections.abc import Iterable from dataclasses import dataclass, field -from typing import List, Set, Union, Dict, Optional +from typing import List, Set, Union, Dict, Optional, FrozenSet from enum import Enum, auto import functools @@ -87,8 +87,22 @@ class SynthInfo: @dataclass -class Class: +class ClassBase: + imported: typing.ClassVar[bool] name: str + + +@dataclass +class ImportedClass(ClassBase): + imported: typing.ClassVar[bool] = True + + module: str + + +@dataclass +class Class(ClassBase): + imported: typing.ClassVar[bool] = False + bases: List[str] = field(default_factory=list) derived: Set[str] = field(default_factory=set) properties: List[Property] = field(default_factory=list) @@ -133,7 +147,7 @@ def group(self) -> str: @dataclass class Schema: - classes: Dict[str, Class] = field(default_factory=dict) + classes: Dict[str, ClassBase] = field(default_factory=dict) includes: List[str] = field(default_factory=list) null: Optional[str] = None @@ -155,7 +169,7 @@ def iter_properties(self, cls: str) -> Iterable[Property]: predicate_marker = object() -TypeRef = Union[type, str] +TypeRef = type | str | ImportedClass def get_type_name(arg: TypeRef) -> str: @@ -164,6 +178,8 @@ def get_type_name(arg: TypeRef) -> str: return arg.__name__ case str(): return arg + case ImportedClass(): + return arg.name case _: raise Error(f"Not a schema type or string ({arg})") @@ -172,9 +188,9 @@ def _make_property(arg: object) -> Property: match arg: case _ if arg is predicate_marker: return PredicateProperty() - case str() | type(): + case (str() | type() | ImportedClass()) as arg: return SingleProperty(type=get_type_name(arg)) - case Property(): + case Property() as arg: return arg case _: raise Error(f"Illegal property specifier {arg}") diff --git a/misc/codegen/lib/schemadefs.py b/misc/codegen/lib/schemadefs.py index 8651240c1a3d..c81b2f2e215a 100644 --- a/misc/codegen/lib/schemadefs.py +++ b/misc/codegen/lib/schemadefs.py @@ -8,8 +8,6 @@ import inspect as _inspect from dataclasses import dataclass as _dataclass -from misc.codegen.lib.schema import Property - _set = set @@ -69,6 +67,9 @@ def include(source: str): _inspect.currentframe().f_back.f_locals.setdefault("includes", []).append(source) +imported = _schema.ImportedClass + + @_dataclass class _Namespace: """ simple namespacing mechanism """ @@ -264,7 +265,7 @@ class _PropertyModifierList(_schema.PropertyModifier): def __or__(self, other: _schema.PropertyModifier): return _PropertyModifierList(self._mods + (other,)) - def modify(self, prop: Property): + def modify(self, prop: _schema.Property): for m in self._mods: m.modify(prop) diff --git a/misc/codegen/loaders/schemaloader.py b/misc/codegen/loaders/schemaloader.py index dd1edee1de09..3b5f20cbbede 100644 --- a/misc/codegen/loaders/schemaloader.py +++ b/misc/codegen/loaders/schemaloader.py @@ -132,6 +132,7 @@ def _check_test_with(classes: typing.Dict[str, schema.Class]): def load(m: types.ModuleType) -> schema.Schema: includes = set() classes = {} + imported_classes = {} known = {"int", "string", "boolean"} known.update(n for n in m.__dict__ if not n.startswith("__")) import misc.codegen.lib.schemadefs as defs @@ -146,6 +147,9 @@ def load(m: types.ModuleType) -> schema.Schema: continue if isinstance(data, types.ModuleType): continue + if isinstance(data, schema.ImportedClass): + imported_classes[name] = data + continue cls = _get_class(data) if classes and not cls.bases: raise schema.Error( @@ -162,7 +166,7 @@ def load(m: types.ModuleType) -> schema.Schema: _fill_hideable_information(classes) _check_test_with(classes) - return schema.Schema(includes=includes, classes=_toposort_classes_by_group(classes), null=null) + return schema.Schema(includes=includes, classes=imported_classes | _toposort_classes_by_group(classes), null=null) def load_file(path: pathlib.Path) -> schema.Schema: diff --git a/misc/codegen/templates/ql_class.mustache b/misc/codegen/templates/ql_class.mustache index da6524d6d375..d39238ff9ba2 100644 --- a/misc/codegen/templates/ql_class.mustache +++ b/misc/codegen/templates/ql_class.mustache @@ -113,7 +113,7 @@ module Generated { */ {{type}} {{getter}}({{#is_indexed}}int index{{/is_indexed}}) { {{^synth}} - {{^is_predicate}}result = {{/is_predicate}}{{#type_is_class}}Synth::convert{{type}}FromRaw({{/type_is_class}}Synth::convert{{name}}ToRaw(this){{^root}}.(Raw::{{name}}){{/root}}.{{getter}}({{#is_indexed}}index{{/is_indexed}}){{#type_is_class}}){{/type_is_class}} + {{^is_predicate}}result = {{/is_predicate}}{{#type_is_codegen_class}}Synth::convert{{type}}FromRaw({{/type_is_codegen_class}}Synth::convert{{name}}ToRaw(this){{^root}}.(Raw::{{name}}){{/root}}.{{getter}}({{#is_indexed}}index{{/is_indexed}}){{#type_is_codegen_class}}){{/type_is_codegen_class}} {{/synth}} {{#synth}} none() diff --git a/misc/codegen/templates/ql_db.mustache b/misc/codegen/templates/ql_db.mustache index 8326bb3adb7d..e63e0aae9034 100644 --- a/misc/codegen/templates/ql_db.mustache +++ b/misc/codegen/templates/ql_db.mustache @@ -3,6 +3,10 @@ * This module holds thin fully generated class definitions around DB entities. */ module Raw { + {{#imports}} + private import {{.}} + {{/imports}} + {{#classes}} /** * INTERNAL: Do not use. diff --git a/misc/codegen/test/test_ql.py b/misc/codegen/test/test_ql.py index eef840ddad6d..e326e65a9e4f 100644 --- a/misc/codegen/test/test_ql.py +++ b/misc/codegen/test/test_ql.py @@ -12,21 +12,6 @@ def test_property_has_first_table_param_marked(): assert [p.param for p in prop.tableparams] == tableparams -@pytest.mark.parametrize("type,expected", [ - ("Foo", True), - ("Bar", True), - ("foo", False), - ("bar", False), - (None, False), -]) -def test_property_is_a_class(type, expected): - tableparams = ["a", "result", "b"] - expected_tableparams = ["a", "result" if expected else "result", "b"] - prop = ql.Property("Prop", type, tableparams=tableparams) - assert prop.type_is_class is expected - assert [p.param for p in prop.tableparams] == expected_tableparams - - indefinite_getters = [ ("Argument", "getAnArgument"), ("Element", "getAnElement"), diff --git a/misc/codegen/test/test_qlgen.py b/misc/codegen/test/test_qlgen.py index 684d3d6a1a10..431f25d5aaeb 100644 --- a/misc/codegen/test/test_qlgen.py +++ b/misc/codegen/test/test_qlgen.py @@ -448,7 +448,8 @@ def test_single_class_property(generate_classes, is_child, prev_child): ql.Property(singular="Foo", type="Bar", tablename="my_objects", tableparams=[ "this", "result"], - prev_child=prev_child, doc="foo of this my object"), + prev_child=prev_child, doc="foo of this my object", + type_is_codegen_class=True), ], )), "Bar.qll": (a_ql_class_public(name="Bar"), a_ql_stub(name="Bar"), a_ql_class(name="Bar", final=True, imports=[stub_import_prefix + "Bar"])), @@ -1006,6 +1007,7 @@ def test_hideable_property(generate_classes): final=True, properties=[ ql.Property(singular="X", type="MyObject", tablename="others", type_is_hideable=True, + type_is_codegen_class=True, tableparams=["this", "result"], doc="x of this other"), ])), } diff --git a/rust/extractor/src/generated/.generated.list b/rust/extractor/src/generated/.generated.list index ec1f28154422..149169eecf09 100644 --- a/rust/extractor/src/generated/.generated.list +++ b/rust/extractor/src/generated/.generated.list @@ -1,2 +1,2 @@ mod.rs 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 -top.rs 272ecf2f56f35211d2449dbf55b1907d8414a8e4cceded03fd12f6f599852c73 272ecf2f56f35211d2449dbf55b1907d8414a8e4cceded03fd12f6f599852c73 +top.rs 0213b8cffeaf38c561be6362078994a150e532f580f0b7a9373252155f595005 0213b8cffeaf38c561be6362078994a150e532f580f0b7a9373252155f595005 diff --git a/rust/extractor/src/generated/top.rs b/rust/extractor/src/generated/top.rs index 73048514fda3..fd6dbadc2890 100644 --- a/rust/extractor/src/generated/top.rs +++ b/rust/extractor/src/generated/top.rs @@ -4,6 +4,15 @@ use crate::trap; +#[derive(Debug)] +pub struct File { + _unused: () +} + +impl trap::TrapClass for File { + fn class_name() -> &'static str { "File" } +} + #[derive(Debug)] pub struct Element { _unused: () @@ -13,6 +22,37 @@ impl trap::TrapClass for Element { fn class_name() -> &'static str { "Element" } } +#[derive(Debug)] +pub struct ExtractorStep { + pub id: trap::TrapId, + pub action: String, + pub file: trap::Label, + pub duration_ms: usize, +} + +impl trap::TrapEntry for ExtractorStep { + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) + } + + fn emit(self, id: trap::Label, out: &mut trap::Writer) { + out.add_tuple("extractor_steps", vec![id.into(), self.action.into(), self.file.into(), self.duration_ms.into()]); + } +} + +impl trap::TrapClass for ExtractorStep { + fn class_name() -> &'static str { "ExtractorStep" } +} + +impl From> for trap::Label { + fn from(value: trap::Label) -> Self { + // SAFETY: this is safe because in the dbscheme ExtractorStep is a subclass of Element + unsafe { + Self::from_untyped(value.as_untyped()) + } + } +} + #[derive(Debug)] pub struct Locatable { _unused: () diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index a3258f16d096..86aeb09f6a4d 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -1,5 +1,6 @@ use crate::diagnostics::{emit_extraction_diagnostics, ExtractionStep}; use crate::rust_analyzer::path_to_file_id; +use crate::trap::TrapId; use anyhow::Context; use archive::Archiver; use log::{info, warn}; @@ -143,21 +144,17 @@ impl<'a> Extractor<'a> { emit_extraction_diagnostics(start, cfg, &self.steps)?; let mut trap = self.traps.create("diagnostics", "extraction"); for step in self.steps { - let file_label = trap.emit_file(&step.file); - let step_label = trap.writer.fresh_id(); - let ms = usize::try_from(step.ms).unwrap_or_else(|_e| { + let file = trap.emit_file(&step.file); + let duration_ms = usize::try_from(step.ms).unwrap_or_else(|_e| { warn!("extraction step duration overflowed ({step:?})"); i32::MAX as usize }); - trap.writer.add_tuple( - "extractor_steps", - vec![ - step_label.into(), - format!("{:?}", step.action).into(), - file_label.into(), - ms.into(), - ], - ); + trap.emit(generated::ExtractorStep { + id: TrapId::Star, + action: format!("{:?}", step.action), + file, + duration_ms, + }); } trap.commit()?; Ok(()) diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 003c86919b67..396816c6a0c8 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -4,7 +4,6 @@ use crate::generated::{self}; use crate::rust_analyzer::FileSemanticInformation; use crate::trap::{DiagnosticSeverity, TrapFile, TrapId}; use crate::trap::{Label, TrapClass}; -use codeql_extractor::trap::{self}; use itertools::Either; use log::Level; use ra_ap_base_db::salsa::InternKey; @@ -65,7 +64,7 @@ macro_rules! emit_detached { pub struct Translator<'a> { pub trap: TrapFile, path: &'a str, - label: trap::Label, + label: Label, line_index: LineIndex, file_id: Option, pub semantics: Option<&'a Semantics<'a, RootDatabase>>, @@ -75,7 +74,7 @@ impl<'a> Translator<'a> { pub fn new( trap: TrapFile, path: &'a str, - label: trap::Label, + label: Label, line_index: LineIndex, semantic_info: Option<&FileSemanticInformation<'a>>, ) -> Translator<'a> { diff --git a/rust/extractor/src/trap.rs b/rust/extractor/src/trap.rs index c993fa046a7c..6c8e9b2c3c84 100644 --- a/rust/extractor/src/trap.rs +++ b/rust/extractor/src/trap.rs @@ -1,5 +1,5 @@ -use crate::config; use crate::config::Compression; +use crate::{config, generated}; use codeql_extractor::{extractor, file_paths, trap}; use log::debug; use ra_ap_ide_db::line_index::LineCol; @@ -138,7 +138,7 @@ pub enum DiagnosticSeverity { impl TrapFile { pub fn emit_location_label( &mut self, - file_label: UntypedLabel, + file_label: Label, start: LineCol, end: LineCol, ) -> UntypedLabel { @@ -149,7 +149,7 @@ impl TrapFile { extractor::location_label( &mut self.writer, trap::Location { - file_label, + file_label: file_label.as_untyped(), start_line, start_column, end_line, @@ -159,7 +159,7 @@ impl TrapFile { } pub fn emit_location( &mut self, - file_label: UntypedLabel, + file_label: Label, entity_label: Label, start: LineCol, end: LineCol, @@ -192,8 +192,10 @@ impl TrapFile { ], ); } - pub fn emit_file(&mut self, absolute_path: &Path) -> trap::Label { - extractor::populate_file(&mut self.writer, absolute_path) + pub fn emit_file(&mut self, absolute_path: &Path) -> Label { + let untyped = extractor::populate_file(&mut self.writer, absolute_path); + // SAFETY: populate_file emits `@file` typed labels + unsafe { Label::from_untyped(untyped) } } pub fn label(&mut self, id: TrapId) -> Label { diff --git a/rust/prefix.dbscheme b/rust/prefix.dbscheme index 5ccbd3438d13..4810e11069d4 100644 --- a/rust/prefix.dbscheme +++ b/rust/prefix.dbscheme @@ -3,10 +3,3 @@ locatable_locations( int id: @locatable ref, int location: @location_default ref ); - -extractor_steps( - unique int id: @extractor_step, - string action: string ref, - int file: @file ref, - int duration_ms: int ref -) diff --git a/rust/ql/.generated.list b/rust/ql/.generated.list index b0c71e3ef015..95e3cd9d349e 100644 --- a/rust/ql/.generated.list +++ b/rust/ql/.generated.list @@ -222,6 +222,8 @@ lib/codeql/rust/elements/internal/ExternCrateImpl.qll ade4df9d3f87daf6534b8e79ff lib/codeql/rust/elements/internal/ExternItemImpl.qll 577c8ac387c47746e3b45f943374c7ab641e8ad119e8591c31f219a5f08d3a29 bba88b974d1c03c78e0caf3d8f4118426d2aa8bd6ffd6f59a3da8ff1524a173f lib/codeql/rust/elements/internal/ExternItemListConstructor.qll 9e4f6a036707c848c0553119272fd2b11c1740dd9910a626a9a0cf68a55b249b efde86b18bd419154fb5b6d28790a14ea989b317d84b5c1ddbdfb29c6924fd86 lib/codeql/rust/elements/internal/ExternItemListImpl.qll e89d0cf938f6e137ba1ce7907a923b1ab2be7be2fdd642c3b7a722f11b9199f8 85906d3ce89e5abc301cc96ea5104d53e90af3f5f22f8d54ec437687096e39d8 +lib/codeql/rust/elements/internal/ExtractorStep.qll 1c65668007ea71d05333e44132eccc01dc2a2b4908fb37d0a73995119d3ed5f0 8cbe1eeb35bc2bc95c1b7765070d1ff58aae03fd28dc94896b091858eea40efe +lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll 00c527a3139ad399ea1efd0ebe4656372d70f6c4e79136bc497a6cb84becae8e 93817f3dddeaf2c0964ab31c2df451dcee0aeba7cb6520803d8ce42cefcb3703 lib/codeql/rust/elements/internal/FieldExprConstructor.qll b3be2c4ccaf2c8a1283f3d5349d7f4f49f87b35e310ef33491023c5ab6f3abc5 645d0d4073b032f6b7284fc36a10a6ec85596fb95c68f30c09504f2c5a6f789f lib/codeql/rust/elements/internal/FieldListImpl.qll 02a09d1d146030c68cead4614f4eef75854f19e55ed1eda60b34c4858a8d4a88 9b9f5e77546434c771d2f785119577ec46569a18473daa4169fb84a097369493 lib/codeql/rust/elements/internal/FnPtrTypeConstructor.qll 494c53ee599039c02145f91394d8dfe7635b32d03f9fcde5efcc99ced437fec8 992462b1b6b9e64b6201f3c6c232ca524f126efcb562c9f0c176677bb559f33c @@ -449,6 +451,7 @@ lib/codeql/rust/elements/internal/generated/ExternBlock.qll c292d804a1f8d2cf6a44 lib/codeql/rust/elements/internal/generated/ExternCrate.qll 35fea4e810a896c1656adb4682c4c3bc20283768073e26ae064189ce310433c8 fc504dff79ba758d89b10cd5049539fbc766ee9862ff495066cea26abf0b5e0b lib/codeql/rust/elements/internal/generated/ExternItem.qll 749b064ad60f32197d5b85e25929afe18e56e12f567b73e21e43e2fdf4c447e3 e2c2d423876675cf2dae399ca442aef7b2860319da9bfadeff29f2c6946f8de7 lib/codeql/rust/elements/internal/generated/ExternItemList.qll 6bc97fdae6c411cab5c501129c1d6c2321c1011cccb119515d75d07dc55c253b 6b5aa808025c0a4270cac540c07ba6faede1b3c70b8db5fd89ec5d46df9041b2 +lib/codeql/rust/elements/internal/generated/ExtractorStep.qll b83ce7f18009bdd36374260652c2a8a5cd5a9b5404a1c147bbec49ad251e43f3 e6e55595300126f9c5a6fd7bde5321b2a0026b491326114d16fcc2395a1fc483 lib/codeql/rust/elements/internal/generated/FieldExpr.qll 3e506b5cb93793ec30f56bb637a600db869fcba6181b068516a671d55c362739 7bbf953696d763ad6b210f378f487ba85b875fa115b22c0c0508599a63633502 lib/codeql/rust/elements/internal/generated/FieldList.qll 43c13c6e3c9ba75a7a4cb870fc4f18752001584d48b9df0734055a6ebb789331 7c51b0b13eb02f1286d3365e53a976ba2655c4dbd8e735bc11c8b205c829e1ee lib/codeql/rust/elements/internal/generated/FnPtrType.qll 748d766dbefd19a7d644734c57885eeede66897029bbfe1b87919517f43bfde2 5a7d80acc00e56594ed85026a8ea4923104d2e98c2e42db8c5bcd32ddd164e48 @@ -512,7 +515,7 @@ lib/codeql/rust/elements/internal/generated/ParamList.qll c808c9d84dd7800573832b lib/codeql/rust/elements/internal/generated/ParenExpr.qll bc0731505bfe88516205ec360582a4222d2681d11342c93e15258590ddee82f2 d4bd6e0c80cf1d63746c88d4bcb3a01d4c75732e5da09e3ebd9437ced227fb60 lib/codeql/rust/elements/internal/generated/ParenPat.qll ce24b8f8ecbf0f204af200317405724063887257460c80cf250c39b2fdf37185 e7c87d37e1a0ca7ea03840017e1aa9ddb7f927f1f3b6396c0305b46aeee33db6 lib/codeql/rust/elements/internal/generated/ParenType.qll 9cc954d73f8330dcac7b475f97748b63af5c8766dee9d2f2872c0a7e4c903537 c07534c8a9c683c4a9b11d490095647e420de0a0bfc23273eaf6f31b00244273 -lib/codeql/rust/elements/internal/generated/ParentChild.qll db7a782f11a14305acc666c865118475e2d324d2bf5d4110b157e1d488b62b75 3b5d31528d0baa0ceee139097e93461d18503797a1507288dc43428f378500e2 +lib/codeql/rust/elements/internal/generated/ParentChild.qll 92402f931bffc872446b7e6d0cc8fd41ff4dc72d0708f9db9b50ac48322d4bc9 2ecc512859db4cf98c680cf948f6dcb89334f999ec58b8d34a0ffd5c385dcef4 lib/codeql/rust/elements/internal/generated/Pat.qll 3605ac062be2f294ee73336e9669027b8b655f4ad55660e1eab35266275154ee 7f9400db2884d336dd1d21df2a8093759c2a110be9bf6482ce8e80ae0fd74ed4 lib/codeql/rust/elements/internal/generated/Path.qll 4c1c8e840ed57880e574142b081b11d7a7428a009f10e3aa8f4645e211f6b2e0 989668cf0f1bdee7557e2f97c01e41d2a56848227fed41477833f5fc1e1d35f6 lib/codeql/rust/elements/internal/generated/PathExpr.qll 2096e3c1db22ee488a761690adabfc9cfdea501c99f7c5d96c0019cb113fc506 54245ce0449c4e263173213df01e079d5168a758503a5dbd61b25ad35a311140 @@ -525,7 +528,7 @@ lib/codeql/rust/elements/internal/generated/PtrType.qll 40099c5a4041314b66932dfd lib/codeql/rust/elements/internal/generated/PureSynthConstructors.qll ea294a3ba33fd1bc632046c4fedbcb84dcb961a8e4599969d65893b19d90e590 ea294a3ba33fd1bc632046c4fedbcb84dcb961a8e4599969d65893b19d90e590 lib/codeql/rust/elements/internal/generated/RangeExpr.qll 23cca03bf43535f33b22a38894f70d669787be4e4f5b8fe5c8f7b964d30e9027 18624cef6c6b679eeace2a98737e472432e0ead354cca02192b4d45330f047c9 lib/codeql/rust/elements/internal/generated/RangePat.qll efd93730de217cf50dcba5875595263a5eadf9f7e4e1272401342a094d158614 229b251b3d118932e31e78ac4dfb75f48b766f240f20d436062785606d44467b -lib/codeql/rust/elements/internal/generated/Raw.qll 7de290d66bd594f4c5b5a296502792e803e9f1084bb2616d9774196e33b16c87 28150fdd3cff3bb49b407f0c2119602be13e78cbb1f8fd749edd31f5d9772f7a +lib/codeql/rust/elements/internal/generated/Raw.qll c302331305186bdf4cfe2aa08395778be84d4959bb535353e1bc9979094e3de2 f40d71d96218cd6dc9b544dfe4aa2d6d035d21ca414ba1cf456741c90076e3db lib/codeql/rust/elements/internal/generated/RecordExpr.qll eb6cb662e463f9260efae1a6ce874fa781172063b916ef1963f861e9942d308d 1a21cbccc8f3799ff13281e822818ebfb21d81591720a427cac3625512cb9d40 lib/codeql/rust/elements/internal/generated/RecordExprField.qll 7e9f8663d3b74ebbc9603b10c9912f082febba6bd73d344b100bbd3edf837802 fbe6b578e7fd5d5a6f21bbb8c388957ab7210a6a249ec71510a50fb35b319ea1 lib/codeql/rust/elements/internal/generated/RecordExprFieldList.qll 179a97211fe7aa6265085d4d54115cdbc0e1cd7c9b2135591e8f36d6432f13d3 dd44bbbc1e83a1ed3a587afb729d7debf7aeb7b63245de181726af13090e50c0 @@ -551,8 +554,8 @@ lib/codeql/rust/elements/internal/generated/Static.qll 5fbd6879858cf356d4bdaa6da lib/codeql/rust/elements/internal/generated/Stmt.qll 8473ff532dd5cc9d7decaddcd174b94d610f6ca0aec8e473cc051dad9f3db917 6ef7d2b5237c2dbdcacbf7d8b39109d4dc100229f2b28b5c9e3e4fbf673ba72b lib/codeql/rust/elements/internal/generated/StmtList.qll a667193e32341e17400867c6e359878c4e645ef9f5f4d97676afc0283a33a026 a320ed678ee359302e2fc1b70a9476705cd616fcfa44a499d32f0c7715627f73 lib/codeql/rust/elements/internal/generated/Struct.qll 4d57f0db12dc7ad3e31e750a24172ef1505406b4dab16386af0674bd18bf8f4b 1a73c83df926b996f629316f74c61ea775be04532ab61b56af904223354f033e -lib/codeql/rust/elements/internal/generated/Synth.qll 65873a7fa44e223edc5e76cc768591a036eb2550960a6b6882476f43a01aefba 3e08e2bdfba53ae26d8f48f2d240b92b44c603f03105518c37a963e0cbe63e3f -lib/codeql/rust/elements/internal/generated/SynthConstructors.qll e929c49ea60810a2bbc19ad38110b8bbaf21db54dae90393b21a3459a54abf6f e929c49ea60810a2bbc19ad38110b8bbaf21db54dae90393b21a3459a54abf6f +lib/codeql/rust/elements/internal/generated/Synth.qll 678edaf70ea0fc2680e53c8cdde0114228be6d8935845f86ccec48274e32d82d 963abedc8cfa60b63db61f250e7da26fc886148408d5340ed500bad3e6dfd7b0 +lib/codeql/rust/elements/internal/generated/SynthConstructors.qll f33931fbdee7e1ca8b5e52af771ea725fef72a2e37fffb628f5270e79d3571a1 f33931fbdee7e1ca8b5e52af771ea725fef72a2e37fffb628f5270e79d3571a1 lib/codeql/rust/elements/internal/generated/Token.qll 77a91a25ca5669703cf3a4353b591cef4d72caa6b0b9db07bb9e005d69c848d1 2fdffc4882ed3a6ca9ac6d1fb5f1ac5a471ca703e2ffdc642885fa558d6e373b lib/codeql/rust/elements/internal/generated/TokenTree.qll 8577c2b097c1be2f0f7daa5acfcf146f78674a424d99563e08a84dd3e6d91b46 d2f30764e84dbfc0a6a5d3d8a5f935cd432413688cb32da9c94e420fbc10665c lib/codeql/rust/elements/internal/generated/Trait.qll 8fa41b50fa0f68333534f2b66bb4ec8e103ff09ac8fa5c2cc64bc04beafec205 ce1c9aa6d0e2f05d28aab8e1165c3b9fb8e24681ade0cf6a9df2e8617abeae7e @@ -586,7 +589,7 @@ lib/codeql/rust/elements/internal/generated/WhileExpr.qll 7edf1f23fbf953a2baabcd lib/codeql/rust/elements/internal/generated/WildcardPat.qll d74b70b57a0a66bfae017a329352a5b27a6b9e73dd5521d627f680e810c6c59e 4b913b548ba27ff3c82fcd32cf996ff329cb57d176d3bebd0fcef394486ea499 lib/codeql/rust/elements/internal/generated/YeetExpr.qll cac328200872a35337b4bcb15c851afb4743f82c080f9738d295571eb01d7392 94af734eea08129b587fed849b643e7572800e8330c0b57d727d41abda47930b lib/codeql/rust/elements/internal/generated/YieldExpr.qll 37e5f0c1e373a22bbc53d8b7f2c0e1f476e5be5080b8437c5e964f4e83fad79a 4a9a68643401637bf48e5c2b2f74a6bf0ddcb4ff76f6bffb61d436b685621e85 -lib/codeql/rust/elements.qll ced76fbeebc6e2e972ecaed65ef97851f90a215cf330f28a0f31a253f1c03442 ced76fbeebc6e2e972ecaed65ef97851f90a215cf330f28a0f31a253f1c03442 +lib/codeql/rust/elements.qll 34cd7db7d8c3cfeac77f23d07418df951158feea426df6ad7d80ebebbcb4e3f2 34cd7db7d8c3cfeac77f23d07418df951158feea426df6ad7d80ebebbcb4e3f2 test/extractor-tests/generated/Abi/Abi.ql 7f6e7dc4af86eca3ebdc79b10373988cd0871bd78b51997d3cffd969105e5fdd 2f936b6ca005c6157c755121584410c03e4a3949c23bee302fbe05ee10ce118f test/extractor-tests/generated/Abi/Abi_getAbiString.ql a496762fcec5a0887b87023bbf93e9b650f02e20113e25c44d6e4281ae8f5335 14109c7ce11ba25e3cd6e7f1b3fcb4cb00622f2a4eac91bfe43145c5f366bc52 test/extractor-tests/generated/ArgList/ArgList.ql e412927756e72165d0e7c5c9bd3fca89d08197bbf760db8fb7683c64bb2229bc 043dba8506946fbb87753e22c387987d7eded6ddb963aa067f9e60ef9024d684 diff --git a/rust/ql/.gitattributes b/rust/ql/.gitattributes index c75ea349c0d1..cca884a14268 100644 --- a/rust/ql/.gitattributes +++ b/rust/ql/.gitattributes @@ -224,6 +224,8 @@ /lib/codeql/rust/elements/internal/ExternItemImpl.qll linguist-generated /lib/codeql/rust/elements/internal/ExternItemListConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/ExternItemListImpl.qll linguist-generated +/lib/codeql/rust/elements/internal/ExtractorStep.qll linguist-generated +/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/FieldExprConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/FieldListImpl.qll linguist-generated /lib/codeql/rust/elements/internal/FnPtrTypeConstructor.qll linguist-generated @@ -451,6 +453,7 @@ /lib/codeql/rust/elements/internal/generated/ExternCrate.qll linguist-generated /lib/codeql/rust/elements/internal/generated/ExternItem.qll linguist-generated /lib/codeql/rust/elements/internal/generated/ExternItemList.qll linguist-generated +/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FieldExpr.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FieldList.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FnPtrType.qll linguist-generated diff --git a/rust/ql/integration-tests/hello-project/steps.ql b/rust/ql/integration-tests/hello-project/steps.ql index eb8f9a05adb7..17358a1c1000 100644 --- a/rust/ql/integration-tests/hello-project/steps.ql +++ b/rust/ql/integration-tests/hello-project/steps.ql @@ -1,4 +1,4 @@ -import codeql.rust.internal.ExtractorStep +import codeql.rust.elements.internal.ExtractorStep from ExtractorStep step select step diff --git a/rust/ql/integration-tests/hello-project/summary.expected b/rust/ql/integration-tests/hello-project/summary.expected index 09e03490eb13..44c14e790fe3 100644 --- a/rust/ql/integration-tests/hello-project/summary.expected +++ b/rust/ql/integration-tests/hello-project/summary.expected @@ -1,4 +1,4 @@ -| Elements extracted | 50 | +| Elements extracted | 65 | | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 1 | diff --git a/rust/ql/integration-tests/hello-workspace/steps.ql b/rust/ql/integration-tests/hello-workspace/steps.ql index eb8f9a05adb7..17358a1c1000 100644 --- a/rust/ql/integration-tests/hello-workspace/steps.ql +++ b/rust/ql/integration-tests/hello-workspace/steps.ql @@ -1,4 +1,4 @@ -import codeql.rust.internal.ExtractorStep +import codeql.rust.elements.internal.ExtractorStep from ExtractorStep step select step diff --git a/rust/ql/integration-tests/hello-workspace/summary.expected b/rust/ql/integration-tests/hello-workspace/summary.cargo.expected similarity index 94% rename from rust/ql/integration-tests/hello-workspace/summary.expected rename to rust/ql/integration-tests/hello-workspace/summary.cargo.expected index 4e800e60bd0c..ec1abea6252b 100644 --- a/rust/ql/integration-tests/hello-workspace/summary.expected +++ b/rust/ql/integration-tests/hello-workspace/summary.cargo.expected @@ -1,4 +1,4 @@ -| Elements extracted | 72 | +| Elements extracted | 86 | | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 0 | diff --git a/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected b/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected new file mode 100644 index 000000000000..5ca38e5a90ce --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected @@ -0,0 +1,17 @@ +| Elements extracted | 85 | +| Elements unextracted | 0 | +| Extraction errors | 0 | +| Extraction warnings | 0 | +| Files extracted - total | 4 | +| Files extracted - with errors | 0 | +| Files extracted - without errors | 4 | +| Inconsistencies - AST | 0 | +| Inconsistencies - CFG | 0 | +| Inconsistencies - data flow | 0 | +| Lines of code extracted | 9 | +| Lines of user code extracted | 9 | +| Macro calls - resolved | 2 | +| Macro calls - total | 2 | +| Macro calls - unresolved | 0 | +| Taint sources - active | 0 | +| Taint sources - total | 0 | diff --git a/rust/ql/integration-tests/hello-workspace/test_workspace.py b/rust/ql/integration-tests/hello-workspace/test_workspace.py index 7f9e3700c3a3..fe8dbc69141c 100644 --- a/rust/ql/integration-tests/hello-workspace/test_workspace.py +++ b/rust/ql/integration-tests/hello-workspace/test_workspace.py @@ -2,11 +2,13 @@ @pytest.mark.ql_test("steps.ql", expected=".cargo.expected") +@pytest.mark.ql_test("summary.qlref", expected=".cargo.expected") def test_cargo(codeql, rust, cargo, check_source_archive, rust_check_diagnostics): rust_check_diagnostics.expected_suffix = ".cargo.expected" codeql.database.create() @pytest.mark.ql_test("steps.ql", expected=".rust-project.expected") +@pytest.mark.ql_test("summary.qlref", expected=".rust-project.expected") def test_rust_project(codeql, rust, rust_project, check_source_archive, rust_check_diagnostics): rust_check_diagnostics.expected_suffix = ".rust-project.expected" codeql.database.create() diff --git a/rust/ql/lib/codeql/files/FileSystem.qll b/rust/ql/lib/codeql/files/FileSystem.qll index c83680f7ec69..b13793b36001 100644 --- a/rust/ql/lib/codeql/files/FileSystem.qll +++ b/rust/ql/lib/codeql/files/FileSystem.qll @@ -6,7 +6,7 @@ private import codeql.rust.elements.SourceFile private import codeql.rust.elements.AstNode private import codeql.rust.elements.Comment private import codeql.rust.Diagnostics -private import codeql.rust.internal.ExtractorStep +private import codeql.rust.elements.internal.ExtractorStep private module Input implements InputSig { abstract class ContainerBase extends @container { diff --git a/rust/ql/lib/codeql/rust/elements.qll b/rust/ql/lib/codeql/rust/elements.qll index e37dde90d618..7954e0e2b14f 100644 --- a/rust/ql/lib/codeql/rust/elements.qll +++ b/rust/ql/lib/codeql/rust/elements.qll @@ -3,6 +3,7 @@ * This module exports all modules providing `Element` subclasses. */ +import codeql.files.FileSystem import codeql.rust.elements.Abi import codeql.rust.elements.Addressable import codeql.rust.elements.ArgList diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll new file mode 100644 index 000000000000..64a4931ef43c --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll @@ -0,0 +1,13 @@ +// generated by codegen, do not edit +/** + * This module provides the class `ExtractorStep`. + */ + +private import ExtractorStepImpl +import codeql.rust.elements.Element +import codeql.files.FileSystem + +/** + * INTERNAL: Do not use. + */ +final class ExtractorStep = Impl::ExtractorStep; diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll new file mode 100644 index 000000000000..76050bf99f86 --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll @@ -0,0 +1,14 @@ +// generated by codegen, remove this comment if you wish to edit this file +/** + * This module defines the hook used internally to tweak the characteristic predicate of + * `ExtractorStep` synthesized instances. + * INTERNAL: Do not use. + */ + +private import codeql.rust.elements.internal.generated.Raw + +/** + * The characteristic predicate of `ExtractorStep` synthesized instances. + * INTERNAL: Do not use. + */ +predicate constructExtractorStep(Raw::ExtractorStep id) { any() } diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll new file mode 100644 index 000000000000..95c677e845e9 --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll @@ -0,0 +1,32 @@ +/** + * This module provides a hand-modifiable wrapper around the generated class `ExtractorStep`. + * + * INTERNAL: Do not use. + */ + +private import codeql.rust.elements.internal.generated.ExtractorStep + +/** + * INTERNAL: This module contains the customizable definition of `ExtractorStep` and should not + * be referenced directly. + */ +module Impl { + class ExtractorStep extends Generated::ExtractorStep { + override string toString() { + result = this.getAction() + "(" + this.getFile().getAbsolutePath() + ")" + } + + /** + * Provides location information for this step. + */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + this.getFile().getAbsolutePath() = filepath and + startline = 0 and + startcolumn = 0 and + endline = 0 and + endcolumn = 0 + } + } +} diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll new file mode 100644 index 000000000000..c2d7838865be --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll @@ -0,0 +1,45 @@ +// generated by codegen, do not edit +/** + * This module provides the generated definition of `ExtractorStep`. + * INTERNAL: Do not import directly. + */ + +private import codeql.rust.elements.internal.generated.Synth +private import codeql.rust.elements.internal.generated.Raw +import codeql.rust.elements.internal.ElementImpl::Impl as ElementImpl +import codeql.files.FileSystem + +/** + * INTERNAL: This module contains the fully generated definition of `ExtractorStep` and should not + * be referenced directly. + */ +module Generated { + /** + * INTERNAL: Do not reference the `Generated::ExtractorStep` class directly. + * Use the subclass `ExtractorStep`, where the following predicates are available. + */ + class ExtractorStep extends Synth::TExtractorStep, ElementImpl::Element { + override string getAPrimaryQlClass() { result = "ExtractorStep" } + + /** + * Gets the action of this extractor step. + */ + string getAction() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getAction() + } + + /** + * Gets the file of this extractor step. + */ + File getFile() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getFile() + } + + /** + * Gets the duration ms of this extractor step. + */ + int getDurationMs() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getDurationMs() + } + } +} diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll index 6709629e8b4f..0ba866f8ee29 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll @@ -4,12 +4,28 @@ */ import codeql.rust.elements +import codeql.rust.elements.internal.ExtractorStep private module Impl { private Element getImmediateChildOfElement(Element e, int index, string partialPredicateCall) { none() } + private Element getImmediateChildOfExtractorStep( + ExtractorStep e, int index, string partialPredicateCall + ) { + exists(int b, int bElement, int n | + b = 0 and + bElement = b + 1 + max(int i | i = -1 or exists(getImmediateChildOfElement(e, i, _)) | i) and + n = bElement and + ( + none() + or + result = getImmediateChildOfElement(e, index - b, partialPredicateCall) + ) + ) + } + private Element getImmediateChildOfLocatable(Locatable e, int index, string partialPredicateCall) { exists(int b, int bElement, int n | b = 0 and @@ -3600,6 +3616,8 @@ private module Impl { // * none() simplifies generation, as we can append `or ...` without a special case for the first item none() or + result = getImmediateChildOfExtractorStep(e, index, partialAccessor) + or result = getImmediateChildOfFormat(e, index, partialAccessor) or result = getImmediateChildOfFormatArgument(e, index, partialAccessor) diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll index 1c85daa22cfa..96db2c99cf6b 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll @@ -3,6 +3,8 @@ * This module holds thin fully generated class definitions around DB entities. */ module Raw { + private import codeql.files.FileSystem + /** * INTERNAL: Do not use. */ @@ -10,6 +12,28 @@ module Raw { string toString() { none() } } + /** + * INTERNAL: Do not use. + */ + class ExtractorStep extends @extractor_step, Element { + override string toString() { result = "ExtractorStep" } + + /** + * Gets the action of this extractor step. + */ + string getAction() { extractor_steps(this, result, _, _) } + + /** + * Gets the file of this extractor step. + */ + File getFile() { extractor_steps(this, _, result, _) } + + /** + * Gets the duration ms of this extractor step. + */ + int getDurationMs() { extractor_steps(this, _, _, result) } + } + /** * INTERNAL: Do not use. */ diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll index 0b4fa39cd848..5284c0b8fde8 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll @@ -134,6 +134,10 @@ module Synth { * INTERNAL: Do not use. */ TExternItemList(Raw::ExternItemList id) { constructExternItemList(id) } or + /** + * INTERNAL: Do not use. + */ + TExtractorStep(Raw::ExtractorStep id) { constructExtractorStep(id) } or /** * INTERNAL: Do not use. */ @@ -926,6 +930,12 @@ module Synth { */ TExternItemList convertExternItemListFromRaw(Raw::Element e) { result = TExternItemList(e) } + /** + * INTERNAL: Do not use. + * Converts a raw element to a synthesized `TExtractorStep`, if possible. + */ + TExtractorStep convertExtractorStepFromRaw(Raw::Element e) { result = TExtractorStep(e) } + /** * INTERNAL: Do not use. * Converts a raw element to a synthesized `TFieldExpr`, if possible. @@ -1803,6 +1813,8 @@ module Synth { * Converts a raw DB element to a synthesized `TElement`, if possible. */ TElement convertElementFromRaw(Raw::Element e) { + result = convertExtractorStepFromRaw(e) + or result = convertLocatableFromRaw(e) or result = convertUnextractedFromRaw(e) @@ -2312,6 +2324,12 @@ module Synth { */ Raw::Element convertExternItemListToRaw(TExternItemList e) { e = TExternItemList(result) } + /** + * INTERNAL: Do not use. + * Converts a synthesized `TExtractorStep` to a raw DB element, if possible. + */ + Raw::Element convertExtractorStepToRaw(TExtractorStep e) { e = TExtractorStep(result) } + /** * INTERNAL: Do not use. * Converts a synthesized `TFieldExpr` to a raw DB element, if possible. @@ -3187,6 +3205,8 @@ module Synth { * Converts a synthesized `TElement` to a raw DB element, if possible. */ Raw::Element convertElementToRaw(TElement e) { + result = convertExtractorStepToRaw(e) + or result = convertLocatableToRaw(e) or result = convertUnextractedToRaw(e) diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll index 9b68c858c6a5..8f7c2c615706 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll @@ -33,6 +33,7 @@ import codeql.rust.elements.internal.ExprStmtConstructor import codeql.rust.elements.internal.ExternBlockConstructor import codeql.rust.elements.internal.ExternCrateConstructor import codeql.rust.elements.internal.ExternItemListConstructor +import codeql.rust.elements.internal.ExtractorStepConstructor import codeql.rust.elements.internal.FieldExprConstructor import codeql.rust.elements.internal.FnPtrTypeConstructor import codeql.rust.elements.internal.ForExprConstructor diff --git a/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll b/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll deleted file mode 100644 index 50e6ebafebfe..000000000000 --- a/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll +++ /dev/null @@ -1,46 +0,0 @@ -import codeql.files.FileSystem - -/** - * An extractor step, usable for debugging and diagnostics reasons. - * - * INTERNAL: Do not use. - */ -class ExtractorStep extends @extractor_step { - /** - * Gets the string representation of this extractor step. - */ - string toString() { - exists(File file, string action | - extractor_steps(this, action, file, _) and - result = action + "(" + file.getAbsolutePath() + ")" - ) - } - - /** - * Gets the action this extractor step carried out. - */ - string getAction() { extractor_steps(this, result, _, _) } - - /** - * Gets the file the extractor step was carried out on. - */ - File getFile() { extractor_steps(this, _, result, _) } - - /** - * Gets the duration of the extractor step in milliseconds. - */ - int getDurationMs() { extractor_steps(this, _, _, result) } - - /** - * Provides location information for this step. - */ - predicate hasLocationInfo( - string filepath, int startline, int startcolumn, int endline, int endcolumn - ) { - this.getFile().getAbsolutePath() = filepath and - startline = 0 and - startcolumn = 0 and - endline = 0 and - endcolumn = 0 - } -} diff --git a/rust/ql/lib/rust.dbscheme b/rust/ql/lib/rust.dbscheme index f0f2a65aea5f..9709ae9bf544 100644 --- a/rust/ql/lib/rust.dbscheme +++ b/rust/ql/lib/rust.dbscheme @@ -116,21 +116,22 @@ locatable_locations( int location: @location_default ref ); -extractor_steps( - unique int id: @extractor_step, - string action: string ref, - int file: @file ref, - int duration_ms: int ref -) - // from schema @element = - @locatable + @extractor_step +| @locatable | @unextracted ; +extractor_steps( + unique int id: @extractor_step, + string action: string ref, + int file: @file ref, + int duration_ms: int ref +); + @locatable = @ast_node ; diff --git a/rust/schema/prelude.py b/rust/schema/prelude.py index ffd65959b5ab..5b050872609b 100644 --- a/rust/schema/prelude.py +++ b/rust/schema/prelude.py @@ -3,6 +3,7 @@ include("../shared/tree-sitter-extractor/src/generator/prefix.dbscheme") include("prefix.dbscheme") +File = imported("File", "codeql.files.FileSystem") @qltest.skip class Element: @@ -93,3 +94,11 @@ class Resolvable(AstNode): """ resolved_path: optional[string] | rust.detach | ql.internal resolved_crate_origin: optional[string] | rust.detach | ql.internal + + +@qltest.skip +@ql.internal +class ExtractorStep(Element): + action: string + file: File + duration_ms: int diff --git a/swift/ql/.generated.list b/swift/ql/.generated.list index da39f2f2fe77..601205793210 100644 --- a/swift/ql/.generated.list +++ b/swift/ql/.generated.list @@ -712,7 +712,7 @@ lib/codeql/swift/generated/OtherAvailabilitySpec.qll d9feaa2a71acff3184ca389045b lib/codeql/swift/generated/ParentChild.qll d1814f2bad4c2ba9242ce49fe6fb8564ac99fc1fd3a7d12aa55e5c6dd7bb529b 1a2075b731d07a5e3c6a69d001796c8de925069d839671a294c9cba6c3db724a lib/codeql/swift/generated/PlatformVersionAvailabilitySpec.qll dc17b49a90a18a8f7607adf2433bc8f0c194fa3e803aa3822f809d4d4fbd6793 be48ea9f8ae17354c8508aaed24337a9e57ce01f288fece3dcecd99776cabcec lib/codeql/swift/generated/PureSynthConstructors.qll bc31a6c4d142fa3fbdcae69d5ba6f1cec00eb9ad92b46c8d7b91ebfa7ef6c1f4 bc31a6c4d142fa3fbdcae69d5ba6f1cec00eb9ad92b46c8d7b91ebfa7ef6c1f4 -lib/codeql/swift/generated/Raw.qll 118b43fedd4265b5aa15c33ef01a2f5a5db6e5597f95bef1078a01c3ff8da983 075aec2c8b232f0361ebf63f07ae9b66163f3975e6023583fb0fa2e40b979a33 +lib/codeql/swift/generated/Raw.qll a47950495630c10c00afee734d779e5e3e7f8e3e65ae3f04b9a1254e3e73921e 075aec2c8b232f0361ebf63f07ae9b66163f3975e6023583fb0fa2e40b979a33 lib/codeql/swift/generated/Synth.qll 31e318c6e156848c85a2a2664695b48b5e93c57c9bb22fa29d027069907b3ab0 8655ffcf772f55284b93f1d7f8e1b3d497a9744d5f2e0c17bc322c1fdf8bdba8 lib/codeql/swift/generated/SynthConstructors.qll 3e53c7853096020219c01dae85681fe80b34938d198a0ff359a209dda41c5ed7 3e53c7853096020219c01dae85681fe80b34938d198a0ff359a209dda41c5ed7 lib/codeql/swift/generated/UnknownFile.qll 247ddf2ebb49ce5ed4bf7bf91a969ddff37de6c78d43d8affccaf7eb586e06f2 452b29f0465ef45e978ef8b647b75e5a2a1e53f2a568fc003bc8f52f73b3fa4d From 4bd5cc458bbf3961449a876bb5eea358005e2ad7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 2 Dec 2024 16:07:00 +0100 Subject: [PATCH 10/10] Rust: accept test changes --- rust/ql/test/query-tests/diagnostics/SummaryStats.expected | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/ql/test/query-tests/diagnostics/SummaryStats.expected b/rust/ql/test/query-tests/diagnostics/SummaryStats.expected index 16e895cdeef3..9372843039cb 100644 --- a/rust/ql/test/query-tests/diagnostics/SummaryStats.expected +++ b/rust/ql/test/query-tests/diagnostics/SummaryStats.expected @@ -1,4 +1,4 @@ -| Elements extracted | 382 | +| Elements extracted | 404 | | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 7 |