From b57a37479b25648a0cdd3925e10a9ed85f779fc7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 2 Dec 2024 15:15:46 +0100 Subject: [PATCH] Rust: make `File` usable in `codegen` --- misc/codegen/generators/dbschemegen.py | 3 +- misc/codegen/generators/qlgen.py | 35 ++++++++++---- misc/codegen/generators/rustgen.py | 16 ++++--- misc/codegen/generators/rusttestgen.py | 2 + misc/codegen/lib/ql.py | 6 +-- misc/codegen/lib/schema.py | 28 ++++++++--- misc/codegen/lib/schemadefs.py | 7 +-- misc/codegen/loaders/schemaloader.py | 6 ++- misc/codegen/templates/ql_class.mustache | 2 +- misc/codegen/templates/ql_db.mustache | 4 ++ misc/codegen/test/test_ql.py | 15 ------ misc/codegen/test/test_qlgen.py | 4 +- rust/extractor/src/generated/.generated.list | 2 +- rust/extractor/src/generated/top.rs | 40 ++++++++++++++++ rust/extractor/src/main.rs | 21 ++++----- rust/extractor/src/translate/base.rs | 5 +- rust/extractor/src/trap.rs | 14 +++--- rust/prefix.dbscheme | 7 --- rust/ql/.generated.list | 13 ++++-- rust/ql/.gitattributes | 3 ++ .../integration-tests/hello-project/steps.ql | 2 +- .../hello-project/summary.expected | 2 +- .../hello-workspace/steps.ql | 2 +- ...ummary.expected => summary.cargo.expected} | 2 +- .../summary.rust-project.expected | 17 +++++++ .../hello-workspace/test_workspace.py | 2 + rust/ql/lib/codeql/files/FileSystem.qll | 2 +- rust/ql/lib/codeql/rust/elements.qll | 1 + .../rust/elements/internal/ExtractorStep.qll | 13 ++++++ .../internal/ExtractorStepConstructor.qll | 14 ++++++ .../elements/internal/ExtractorStepImpl.qll | 32 +++++++++++++ .../internal/generated/ExtractorStep.qll | 45 ++++++++++++++++++ .../internal/generated/ParentChild.qll | 18 ++++++++ .../rust/elements/internal/generated/Raw.qll | 24 ++++++++++ .../elements/internal/generated/Synth.qll | 20 ++++++++ .../internal/generated/SynthConstructors.qll | 1 + .../codeql/rust/internal/ExtractorStep.qll | 46 ------------------- rust/ql/lib/rust.dbscheme | 17 +++---- rust/schema/prelude.py | 9 ++++ swift/ql/.generated.list | 2 +- 40 files changed, 363 insertions(+), 141 deletions(-) rename rust/ql/integration-tests/hello-workspace/{summary.expected => summary.cargo.expected} (94%) create mode 100644 rust/ql/integration-tests/hello-workspace/summary.rust-project.expected create mode 100644 rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll create mode 100644 rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll create mode 100644 rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll create mode 100644 rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll delete mode 100644 rust/ql/lib/codeql/rust/internal/ExtractorStep.qll diff --git a/misc/codegen/generators/dbschemegen.py b/misc/codegen/generators/dbschemegen.py index 2c3cd5598d55..e2cc4220dc71 100755 --- a/misc/codegen/generators/dbschemegen.py +++ b/misc/codegen/generators/dbschemegen.py @@ -110,7 +110,8 @@ def cls_to_dbscheme(cls: schema.Class, lookup: typing.Dict[str, schema.Class], a def get_declarations(data: schema.Schema): add_or_none_except = data.root_class.name if data.null else None - declarations = [d for cls in data.classes.values() for d in cls_to_dbscheme(cls, data.classes, add_or_none_except)] + declarations = [d for cls in data.classes.values() if not cls.imported for d in cls_to_dbscheme(cls, + data.classes, add_or_none_except)] if data.null: property_classes = { prop.type for cls in data.classes.values() for prop in cls.properties diff --git a/misc/codegen/generators/qlgen.py b/misc/codegen/generators/qlgen.py index e42c9d015522..feb2250bf3bb 100755 --- a/misc/codegen/generators/qlgen.py +++ b/misc/codegen/generators/qlgen.py @@ -104,8 +104,17 @@ def _get_doc(cls: schema.Class, prop: schema.Property, plural=None): return f"{prop_name} of this {class_name}" -def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.Class], +def _type_is_hideable(t: str, lookup: typing.Dict[str, schema.ClassBase]) -> bool: + if t in lookup: + match lookup[t]: + case schema.Class() as cls: + return "ql_hideable" in cls.pragmas + return False + + +def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.ClassBase], prev_child: str = "") -> ql.Property: + args = dict( type=prop.type if not prop.is_predicate else "predicate", qltest_skip="qltest_skip" in prop.pragmas, @@ -115,7 +124,8 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic is_unordered=prop.is_unordered, description=prop.description, synth=bool(cls.synth) or prop.synth, - type_is_hideable="ql_hideable" in lookup[prop.type].pragmas if prop.type in lookup else False, + type_is_hideable=_type_is_hideable(prop.type, lookup), + type_is_codegen_class=prop.type in lookup and not lookup[prop.type].imported, internal="ql_internal" in prop.pragmas, ) ql_name = prop.pragmas.get("ql_name", prop.name) @@ -154,7 +164,7 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic return ql.Property(**args) -def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]) -> ql.Class: +def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.ClassBase]) -> ql.Class: if "ql_name" in cls.pragmas: raise Error("ql_name is not supported yet for classes, only for properties") prev_child = "" @@ -391,14 +401,15 @@ def generate(opts, renderer): data = schemaloader.load_file(input) - classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items()} + classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items() if not cls.imported} if not classes: raise NoClasses root = next(iter(classes.values())) if root.has_children: raise RootElementHasChildren(root) - imports = {} + pre_imports = {n: cls.module for n, cls in data.classes.items() if cls.imported} + imports = dict(pre_imports) imports_impl = {} classes_used_by = {} cfg_classes = [] @@ -410,7 +421,7 @@ def generate(opts, renderer): force=opts.force) as renderer: db_classes = [cls for name, cls in classes.items() if not data.classes[name].synth] - renderer.render(ql.DbClasses(db_classes), out / "Raw.qll") + renderer.render(ql.DbClasses(classes=db_classes, imports=sorted(set(pre_imports.values()))), out / "Raw.qll") classes_by_dir_and_name = sorted(classes.values(), key=lambda cls: (cls.dir, cls.name)) for c in classes_by_dir_and_name: @@ -439,6 +450,8 @@ def generate(opts, renderer): renderer.render(cfg_classes_val, cfg_qll) for c in data.classes.values(): + if c.imported: + continue path = _get_path(c) path_impl = _get_path_impl(c) stub_file = stub_out / path_impl @@ -457,7 +470,7 @@ def generate(opts, renderer): renderer.render(class_public, class_public_file) # for example path/to/elements -> path/to/elements.qll - renderer.render(ql.ImportList([i for name, i in imports.items() if not classes[name].internal]), + renderer.render(ql.ImportList([i for name, i in imports.items() if name not in classes or not classes[name].internal]), include_file) elements_module = get_import(include_file, opts.root_dir) @@ -465,12 +478,15 @@ def generate(opts, renderer): renderer.render( ql.GetParentImplementation( classes=list(classes.values()), - imports=[elements_module] + [i for name, i in imports.items() if classes[name].internal], + imports=[elements_module] + [i for name, + i in imports.items() if name in classes and classes[name].internal], ), out / 'ParentChild.qll') if test_out: for c in data.classes.values(): + if c.imported: + continue if should_skip_qltest(c, data.classes): continue test_with_name = c.pragmas.get("qltest_test_with") @@ -500,7 +516,8 @@ def generate(opts, renderer): constructor_imports = [] synth_constructor_imports = [] stubs = {} - for cls in sorted(data.classes.values(), key=lambda cls: (cls.group, cls.name)): + for cls in sorted((cls for cls in data.classes.values() if not cls.imported), + key=lambda cls: (cls.group, cls.name)): synth_type = get_ql_synth_class(cls) if synth_type.is_final: final_synth_types.append(synth_type) diff --git a/misc/codegen/generators/rustgen.py b/misc/codegen/generators/rustgen.py index f4c977c0fdbe..b47e5cc4bd9d 100644 --- a/misc/codegen/generators/rustgen.py +++ b/misc/codegen/generators/rustgen.py @@ -49,7 +49,7 @@ def _get_field(cls: schema.Class, p: schema.Property) -> rust.Field: def _get_properties( - cls: schema.Class, lookup: dict[str, schema.Class], + cls: schema.Class, lookup: dict[str, schema.ClassBase], ) -> typing.Iterable[tuple[schema.Class, schema.Property]]: for b in cls.bases: yield from _get_properties(lookup[b], lookup) @@ -58,12 +58,14 @@ def _get_properties( def _get_ancestors( - cls: schema.Class, lookup: dict[str, schema.Class] + cls: schema.Class, lookup: dict[str, schema.ClassBase] ) -> typing.Iterable[schema.Class]: for b in cls.bases: base = lookup[b] - yield base - yield from _get_ancestors(base, lookup) + if not base.imported: + base = typing.cast(schema.Class, base) + yield base + yield from _get_ancestors(base, lookup) class Processor: @@ -71,7 +73,7 @@ def __init__(self, data: schema.Schema): self._classmap = data.classes def _get_class(self, name: str) -> rust.Class: - cls = self._classmap[name] + cls = typing.cast(schema.Class, self._classmap[name]) properties = [ (c, p) for c, p in _get_properties(cls, self._classmap) @@ -101,8 +103,10 @@ def _get_class(self, name: str) -> rust.Class: def get_classes(self): ret = {"": []} for k, cls in self._classmap.items(): - if not cls.synth: + if not cls.imported and not cls.synth: ret.setdefault(cls.group, []).append(self._get_class(cls.name)) + elif cls.imported: + ret[""].append(rust.Class(name=cls.name)) return ret diff --git a/misc/codegen/generators/rusttestgen.py b/misc/codegen/generators/rusttestgen.py index d360db27a658..e7a23fedacdc 100644 --- a/misc/codegen/generators/rusttestgen.py +++ b/misc/codegen/generators/rusttestgen.py @@ -56,6 +56,8 @@ def generate(opts, renderer): registry=opts.ql_test_output / ".generated_tests.list", force=opts.force) as renderer: for cls in schema.classes.values(): + if cls.imported: + continue if (qlgen.should_skip_qltest(cls, schema.classes) or "rust_skip_doc_test" in cls.pragmas): continue diff --git a/misc/codegen/lib/ql.py b/misc/codegen/lib/ql.py index 1182c7f5dc16..b9362a556ef1 100644 --- a/misc/codegen/lib/ql.py +++ b/misc/codegen/lib/ql.py @@ -44,6 +44,7 @@ class Property: doc_plural: Optional[str] = None synth: bool = False type_is_hideable: bool = False + type_is_codegen_class: bool = False internal: bool = False cfg: bool = False @@ -66,10 +67,6 @@ def indefinite_getter(self): article = "An" if self.singular[0] in "AEIO" else "A" return f"get{article}{self.singular}" - @property - def type_is_class(self): - return bool(self.type) and self.type[0].isupper() - @property def is_repeated(self): return bool(self.plural) @@ -191,6 +188,7 @@ class DbClasses: template: ClassVar = 'ql_db' classes: List[Class] = field(default_factory=list) + imports: List[str] = field(default_factory=list) @dataclass diff --git a/misc/codegen/lib/schema.py b/misc/codegen/lib/schema.py index 23f1aea2ba42..5178e61d3844 100644 --- a/misc/codegen/lib/schema.py +++ b/misc/codegen/lib/schema.py @@ -3,7 +3,7 @@ import typing from collections.abc import Iterable from dataclasses import dataclass, field -from typing import List, Set, Union, Dict, Optional +from typing import List, Set, Union, Dict, Optional, FrozenSet from enum import Enum, auto import functools @@ -87,8 +87,22 @@ class SynthInfo: @dataclass -class Class: +class ClassBase: + imported: typing.ClassVar[bool] name: str + + +@dataclass +class ImportedClass(ClassBase): + imported: typing.ClassVar[bool] = True + + module: str + + +@dataclass +class Class(ClassBase): + imported: typing.ClassVar[bool] = False + bases: List[str] = field(default_factory=list) derived: Set[str] = field(default_factory=set) properties: List[Property] = field(default_factory=list) @@ -133,7 +147,7 @@ def group(self) -> str: @dataclass class Schema: - classes: Dict[str, Class] = field(default_factory=dict) + classes: Dict[str, ClassBase] = field(default_factory=dict) includes: List[str] = field(default_factory=list) null: Optional[str] = None @@ -155,7 +169,7 @@ def iter_properties(self, cls: str) -> Iterable[Property]: predicate_marker = object() -TypeRef = Union[type, str] +TypeRef = type | str | ImportedClass def get_type_name(arg: TypeRef) -> str: @@ -164,6 +178,8 @@ def get_type_name(arg: TypeRef) -> str: return arg.__name__ case str(): return arg + case ImportedClass(): + return arg.name case _: raise Error(f"Not a schema type or string ({arg})") @@ -172,9 +188,9 @@ def _make_property(arg: object) -> Property: match arg: case _ if arg is predicate_marker: return PredicateProperty() - case str() | type(): + case (str() | type() | ImportedClass()) as arg: return SingleProperty(type=get_type_name(arg)) - case Property(): + case Property() as arg: return arg case _: raise Error(f"Illegal property specifier {arg}") diff --git a/misc/codegen/lib/schemadefs.py b/misc/codegen/lib/schemadefs.py index 8651240c1a3d..c81b2f2e215a 100644 --- a/misc/codegen/lib/schemadefs.py +++ b/misc/codegen/lib/schemadefs.py @@ -8,8 +8,6 @@ import inspect as _inspect from dataclasses import dataclass as _dataclass -from misc.codegen.lib.schema import Property - _set = set @@ -69,6 +67,9 @@ def include(source: str): _inspect.currentframe().f_back.f_locals.setdefault("includes", []).append(source) +imported = _schema.ImportedClass + + @_dataclass class _Namespace: """ simple namespacing mechanism """ @@ -264,7 +265,7 @@ class _PropertyModifierList(_schema.PropertyModifier): def __or__(self, other: _schema.PropertyModifier): return _PropertyModifierList(self._mods + (other,)) - def modify(self, prop: Property): + def modify(self, prop: _schema.Property): for m in self._mods: m.modify(prop) diff --git a/misc/codegen/loaders/schemaloader.py b/misc/codegen/loaders/schemaloader.py index dd1edee1de09..3b5f20cbbede 100644 --- a/misc/codegen/loaders/schemaloader.py +++ b/misc/codegen/loaders/schemaloader.py @@ -132,6 +132,7 @@ def _check_test_with(classes: typing.Dict[str, schema.Class]): def load(m: types.ModuleType) -> schema.Schema: includes = set() classes = {} + imported_classes = {} known = {"int", "string", "boolean"} known.update(n for n in m.__dict__ if not n.startswith("__")) import misc.codegen.lib.schemadefs as defs @@ -146,6 +147,9 @@ def load(m: types.ModuleType) -> schema.Schema: continue if isinstance(data, types.ModuleType): continue + if isinstance(data, schema.ImportedClass): + imported_classes[name] = data + continue cls = _get_class(data) if classes and not cls.bases: raise schema.Error( @@ -162,7 +166,7 @@ def load(m: types.ModuleType) -> schema.Schema: _fill_hideable_information(classes) _check_test_with(classes) - return schema.Schema(includes=includes, classes=_toposort_classes_by_group(classes), null=null) + return schema.Schema(includes=includes, classes=imported_classes | _toposort_classes_by_group(classes), null=null) def load_file(path: pathlib.Path) -> schema.Schema: diff --git a/misc/codegen/templates/ql_class.mustache b/misc/codegen/templates/ql_class.mustache index da6524d6d375..d39238ff9ba2 100644 --- a/misc/codegen/templates/ql_class.mustache +++ b/misc/codegen/templates/ql_class.mustache @@ -113,7 +113,7 @@ module Generated { */ {{type}} {{getter}}({{#is_indexed}}int index{{/is_indexed}}) { {{^synth}} - {{^is_predicate}}result = {{/is_predicate}}{{#type_is_class}}Synth::convert{{type}}FromRaw({{/type_is_class}}Synth::convert{{name}}ToRaw(this){{^root}}.(Raw::{{name}}){{/root}}.{{getter}}({{#is_indexed}}index{{/is_indexed}}){{#type_is_class}}){{/type_is_class}} + {{^is_predicate}}result = {{/is_predicate}}{{#type_is_codegen_class}}Synth::convert{{type}}FromRaw({{/type_is_codegen_class}}Synth::convert{{name}}ToRaw(this){{^root}}.(Raw::{{name}}){{/root}}.{{getter}}({{#is_indexed}}index{{/is_indexed}}){{#type_is_codegen_class}}){{/type_is_codegen_class}} {{/synth}} {{#synth}} none() diff --git a/misc/codegen/templates/ql_db.mustache b/misc/codegen/templates/ql_db.mustache index 8326bb3adb7d..e63e0aae9034 100644 --- a/misc/codegen/templates/ql_db.mustache +++ b/misc/codegen/templates/ql_db.mustache @@ -3,6 +3,10 @@ * This module holds thin fully generated class definitions around DB entities. */ module Raw { + {{#imports}} + private import {{.}} + {{/imports}} + {{#classes}} /** * INTERNAL: Do not use. diff --git a/misc/codegen/test/test_ql.py b/misc/codegen/test/test_ql.py index eef840ddad6d..e326e65a9e4f 100644 --- a/misc/codegen/test/test_ql.py +++ b/misc/codegen/test/test_ql.py @@ -12,21 +12,6 @@ def test_property_has_first_table_param_marked(): assert [p.param for p in prop.tableparams] == tableparams -@pytest.mark.parametrize("type,expected", [ - ("Foo", True), - ("Bar", True), - ("foo", False), - ("bar", False), - (None, False), -]) -def test_property_is_a_class(type, expected): - tableparams = ["a", "result", "b"] - expected_tableparams = ["a", "result" if expected else "result", "b"] - prop = ql.Property("Prop", type, tableparams=tableparams) - assert prop.type_is_class is expected - assert [p.param for p in prop.tableparams] == expected_tableparams - - indefinite_getters = [ ("Argument", "getAnArgument"), ("Element", "getAnElement"), diff --git a/misc/codegen/test/test_qlgen.py b/misc/codegen/test/test_qlgen.py index 684d3d6a1a10..431f25d5aaeb 100644 --- a/misc/codegen/test/test_qlgen.py +++ b/misc/codegen/test/test_qlgen.py @@ -448,7 +448,8 @@ def test_single_class_property(generate_classes, is_child, prev_child): ql.Property(singular="Foo", type="Bar", tablename="my_objects", tableparams=[ "this", "result"], - prev_child=prev_child, doc="foo of this my object"), + prev_child=prev_child, doc="foo of this my object", + type_is_codegen_class=True), ], )), "Bar.qll": (a_ql_class_public(name="Bar"), a_ql_stub(name="Bar"), a_ql_class(name="Bar", final=True, imports=[stub_import_prefix + "Bar"])), @@ -1006,6 +1007,7 @@ def test_hideable_property(generate_classes): final=True, properties=[ ql.Property(singular="X", type="MyObject", tablename="others", type_is_hideable=True, + type_is_codegen_class=True, tableparams=["this", "result"], doc="x of this other"), ])), } diff --git a/rust/extractor/src/generated/.generated.list b/rust/extractor/src/generated/.generated.list index ec1f28154422..149169eecf09 100644 --- a/rust/extractor/src/generated/.generated.list +++ b/rust/extractor/src/generated/.generated.list @@ -1,2 +1,2 @@ mod.rs 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 -top.rs 272ecf2f56f35211d2449dbf55b1907d8414a8e4cceded03fd12f6f599852c73 272ecf2f56f35211d2449dbf55b1907d8414a8e4cceded03fd12f6f599852c73 +top.rs 0213b8cffeaf38c561be6362078994a150e532f580f0b7a9373252155f595005 0213b8cffeaf38c561be6362078994a150e532f580f0b7a9373252155f595005 diff --git a/rust/extractor/src/generated/top.rs b/rust/extractor/src/generated/top.rs index 73048514fda3..fd6dbadc2890 100644 --- a/rust/extractor/src/generated/top.rs +++ b/rust/extractor/src/generated/top.rs @@ -4,6 +4,15 @@ use crate::trap; +#[derive(Debug)] +pub struct File { + _unused: () +} + +impl trap::TrapClass for File { + fn class_name() -> &'static str { "File" } +} + #[derive(Debug)] pub struct Element { _unused: () @@ -13,6 +22,37 @@ impl trap::TrapClass for Element { fn class_name() -> &'static str { "Element" } } +#[derive(Debug)] +pub struct ExtractorStep { + pub id: trap::TrapId, + pub action: String, + pub file: trap::Label, + pub duration_ms: usize, +} + +impl trap::TrapEntry for ExtractorStep { + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) + } + + fn emit(self, id: trap::Label, out: &mut trap::Writer) { + out.add_tuple("extractor_steps", vec![id.into(), self.action.into(), self.file.into(), self.duration_ms.into()]); + } +} + +impl trap::TrapClass for ExtractorStep { + fn class_name() -> &'static str { "ExtractorStep" } +} + +impl From> for trap::Label { + fn from(value: trap::Label) -> Self { + // SAFETY: this is safe because in the dbscheme ExtractorStep is a subclass of Element + unsafe { + Self::from_untyped(value.as_untyped()) + } + } +} + #[derive(Debug)] pub struct Locatable { _unused: () diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index a3258f16d096..86aeb09f6a4d 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -1,5 +1,6 @@ use crate::diagnostics::{emit_extraction_diagnostics, ExtractionStep}; use crate::rust_analyzer::path_to_file_id; +use crate::trap::TrapId; use anyhow::Context; use archive::Archiver; use log::{info, warn}; @@ -143,21 +144,17 @@ impl<'a> Extractor<'a> { emit_extraction_diagnostics(start, cfg, &self.steps)?; let mut trap = self.traps.create("diagnostics", "extraction"); for step in self.steps { - let file_label = trap.emit_file(&step.file); - let step_label = trap.writer.fresh_id(); - let ms = usize::try_from(step.ms).unwrap_or_else(|_e| { + let file = trap.emit_file(&step.file); + let duration_ms = usize::try_from(step.ms).unwrap_or_else(|_e| { warn!("extraction step duration overflowed ({step:?})"); i32::MAX as usize }); - trap.writer.add_tuple( - "extractor_steps", - vec![ - step_label.into(), - format!("{:?}", step.action).into(), - file_label.into(), - ms.into(), - ], - ); + trap.emit(generated::ExtractorStep { + id: TrapId::Star, + action: format!("{:?}", step.action), + file, + duration_ms, + }); } trap.commit()?; Ok(()) diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 003c86919b67..396816c6a0c8 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -4,7 +4,6 @@ use crate::generated::{self}; use crate::rust_analyzer::FileSemanticInformation; use crate::trap::{DiagnosticSeverity, TrapFile, TrapId}; use crate::trap::{Label, TrapClass}; -use codeql_extractor::trap::{self}; use itertools::Either; use log::Level; use ra_ap_base_db::salsa::InternKey; @@ -65,7 +64,7 @@ macro_rules! emit_detached { pub struct Translator<'a> { pub trap: TrapFile, path: &'a str, - label: trap::Label, + label: Label, line_index: LineIndex, file_id: Option, pub semantics: Option<&'a Semantics<'a, RootDatabase>>, @@ -75,7 +74,7 @@ impl<'a> Translator<'a> { pub fn new( trap: TrapFile, path: &'a str, - label: trap::Label, + label: Label, line_index: LineIndex, semantic_info: Option<&FileSemanticInformation<'a>>, ) -> Translator<'a> { diff --git a/rust/extractor/src/trap.rs b/rust/extractor/src/trap.rs index c993fa046a7c..6c8e9b2c3c84 100644 --- a/rust/extractor/src/trap.rs +++ b/rust/extractor/src/trap.rs @@ -1,5 +1,5 @@ -use crate::config; use crate::config::Compression; +use crate::{config, generated}; use codeql_extractor::{extractor, file_paths, trap}; use log::debug; use ra_ap_ide_db::line_index::LineCol; @@ -138,7 +138,7 @@ pub enum DiagnosticSeverity { impl TrapFile { pub fn emit_location_label( &mut self, - file_label: UntypedLabel, + file_label: Label, start: LineCol, end: LineCol, ) -> UntypedLabel { @@ -149,7 +149,7 @@ impl TrapFile { extractor::location_label( &mut self.writer, trap::Location { - file_label, + file_label: file_label.as_untyped(), start_line, start_column, end_line, @@ -159,7 +159,7 @@ impl TrapFile { } pub fn emit_location( &mut self, - file_label: UntypedLabel, + file_label: Label, entity_label: Label, start: LineCol, end: LineCol, @@ -192,8 +192,10 @@ impl TrapFile { ], ); } - pub fn emit_file(&mut self, absolute_path: &Path) -> trap::Label { - extractor::populate_file(&mut self.writer, absolute_path) + pub fn emit_file(&mut self, absolute_path: &Path) -> Label { + let untyped = extractor::populate_file(&mut self.writer, absolute_path); + // SAFETY: populate_file emits `@file` typed labels + unsafe { Label::from_untyped(untyped) } } pub fn label(&mut self, id: TrapId) -> Label { diff --git a/rust/prefix.dbscheme b/rust/prefix.dbscheme index 5ccbd3438d13..4810e11069d4 100644 --- a/rust/prefix.dbscheme +++ b/rust/prefix.dbscheme @@ -3,10 +3,3 @@ locatable_locations( int id: @locatable ref, int location: @location_default ref ); - -extractor_steps( - unique int id: @extractor_step, - string action: string ref, - int file: @file ref, - int duration_ms: int ref -) diff --git a/rust/ql/.generated.list b/rust/ql/.generated.list index b0c71e3ef015..95e3cd9d349e 100644 --- a/rust/ql/.generated.list +++ b/rust/ql/.generated.list @@ -222,6 +222,8 @@ lib/codeql/rust/elements/internal/ExternCrateImpl.qll ade4df9d3f87daf6534b8e79ff lib/codeql/rust/elements/internal/ExternItemImpl.qll 577c8ac387c47746e3b45f943374c7ab641e8ad119e8591c31f219a5f08d3a29 bba88b974d1c03c78e0caf3d8f4118426d2aa8bd6ffd6f59a3da8ff1524a173f lib/codeql/rust/elements/internal/ExternItemListConstructor.qll 9e4f6a036707c848c0553119272fd2b11c1740dd9910a626a9a0cf68a55b249b efde86b18bd419154fb5b6d28790a14ea989b317d84b5c1ddbdfb29c6924fd86 lib/codeql/rust/elements/internal/ExternItemListImpl.qll e89d0cf938f6e137ba1ce7907a923b1ab2be7be2fdd642c3b7a722f11b9199f8 85906d3ce89e5abc301cc96ea5104d53e90af3f5f22f8d54ec437687096e39d8 +lib/codeql/rust/elements/internal/ExtractorStep.qll 1c65668007ea71d05333e44132eccc01dc2a2b4908fb37d0a73995119d3ed5f0 8cbe1eeb35bc2bc95c1b7765070d1ff58aae03fd28dc94896b091858eea40efe +lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll 00c527a3139ad399ea1efd0ebe4656372d70f6c4e79136bc497a6cb84becae8e 93817f3dddeaf2c0964ab31c2df451dcee0aeba7cb6520803d8ce42cefcb3703 lib/codeql/rust/elements/internal/FieldExprConstructor.qll b3be2c4ccaf2c8a1283f3d5349d7f4f49f87b35e310ef33491023c5ab6f3abc5 645d0d4073b032f6b7284fc36a10a6ec85596fb95c68f30c09504f2c5a6f789f lib/codeql/rust/elements/internal/FieldListImpl.qll 02a09d1d146030c68cead4614f4eef75854f19e55ed1eda60b34c4858a8d4a88 9b9f5e77546434c771d2f785119577ec46569a18473daa4169fb84a097369493 lib/codeql/rust/elements/internal/FnPtrTypeConstructor.qll 494c53ee599039c02145f91394d8dfe7635b32d03f9fcde5efcc99ced437fec8 992462b1b6b9e64b6201f3c6c232ca524f126efcb562c9f0c176677bb559f33c @@ -449,6 +451,7 @@ lib/codeql/rust/elements/internal/generated/ExternBlock.qll c292d804a1f8d2cf6a44 lib/codeql/rust/elements/internal/generated/ExternCrate.qll 35fea4e810a896c1656adb4682c4c3bc20283768073e26ae064189ce310433c8 fc504dff79ba758d89b10cd5049539fbc766ee9862ff495066cea26abf0b5e0b lib/codeql/rust/elements/internal/generated/ExternItem.qll 749b064ad60f32197d5b85e25929afe18e56e12f567b73e21e43e2fdf4c447e3 e2c2d423876675cf2dae399ca442aef7b2860319da9bfadeff29f2c6946f8de7 lib/codeql/rust/elements/internal/generated/ExternItemList.qll 6bc97fdae6c411cab5c501129c1d6c2321c1011cccb119515d75d07dc55c253b 6b5aa808025c0a4270cac540c07ba6faede1b3c70b8db5fd89ec5d46df9041b2 +lib/codeql/rust/elements/internal/generated/ExtractorStep.qll b83ce7f18009bdd36374260652c2a8a5cd5a9b5404a1c147bbec49ad251e43f3 e6e55595300126f9c5a6fd7bde5321b2a0026b491326114d16fcc2395a1fc483 lib/codeql/rust/elements/internal/generated/FieldExpr.qll 3e506b5cb93793ec30f56bb637a600db869fcba6181b068516a671d55c362739 7bbf953696d763ad6b210f378f487ba85b875fa115b22c0c0508599a63633502 lib/codeql/rust/elements/internal/generated/FieldList.qll 43c13c6e3c9ba75a7a4cb870fc4f18752001584d48b9df0734055a6ebb789331 7c51b0b13eb02f1286d3365e53a976ba2655c4dbd8e735bc11c8b205c829e1ee lib/codeql/rust/elements/internal/generated/FnPtrType.qll 748d766dbefd19a7d644734c57885eeede66897029bbfe1b87919517f43bfde2 5a7d80acc00e56594ed85026a8ea4923104d2e98c2e42db8c5bcd32ddd164e48 @@ -512,7 +515,7 @@ lib/codeql/rust/elements/internal/generated/ParamList.qll c808c9d84dd7800573832b lib/codeql/rust/elements/internal/generated/ParenExpr.qll bc0731505bfe88516205ec360582a4222d2681d11342c93e15258590ddee82f2 d4bd6e0c80cf1d63746c88d4bcb3a01d4c75732e5da09e3ebd9437ced227fb60 lib/codeql/rust/elements/internal/generated/ParenPat.qll ce24b8f8ecbf0f204af200317405724063887257460c80cf250c39b2fdf37185 e7c87d37e1a0ca7ea03840017e1aa9ddb7f927f1f3b6396c0305b46aeee33db6 lib/codeql/rust/elements/internal/generated/ParenType.qll 9cc954d73f8330dcac7b475f97748b63af5c8766dee9d2f2872c0a7e4c903537 c07534c8a9c683c4a9b11d490095647e420de0a0bfc23273eaf6f31b00244273 -lib/codeql/rust/elements/internal/generated/ParentChild.qll db7a782f11a14305acc666c865118475e2d324d2bf5d4110b157e1d488b62b75 3b5d31528d0baa0ceee139097e93461d18503797a1507288dc43428f378500e2 +lib/codeql/rust/elements/internal/generated/ParentChild.qll 92402f931bffc872446b7e6d0cc8fd41ff4dc72d0708f9db9b50ac48322d4bc9 2ecc512859db4cf98c680cf948f6dcb89334f999ec58b8d34a0ffd5c385dcef4 lib/codeql/rust/elements/internal/generated/Pat.qll 3605ac062be2f294ee73336e9669027b8b655f4ad55660e1eab35266275154ee 7f9400db2884d336dd1d21df2a8093759c2a110be9bf6482ce8e80ae0fd74ed4 lib/codeql/rust/elements/internal/generated/Path.qll 4c1c8e840ed57880e574142b081b11d7a7428a009f10e3aa8f4645e211f6b2e0 989668cf0f1bdee7557e2f97c01e41d2a56848227fed41477833f5fc1e1d35f6 lib/codeql/rust/elements/internal/generated/PathExpr.qll 2096e3c1db22ee488a761690adabfc9cfdea501c99f7c5d96c0019cb113fc506 54245ce0449c4e263173213df01e079d5168a758503a5dbd61b25ad35a311140 @@ -525,7 +528,7 @@ lib/codeql/rust/elements/internal/generated/PtrType.qll 40099c5a4041314b66932dfd lib/codeql/rust/elements/internal/generated/PureSynthConstructors.qll ea294a3ba33fd1bc632046c4fedbcb84dcb961a8e4599969d65893b19d90e590 ea294a3ba33fd1bc632046c4fedbcb84dcb961a8e4599969d65893b19d90e590 lib/codeql/rust/elements/internal/generated/RangeExpr.qll 23cca03bf43535f33b22a38894f70d669787be4e4f5b8fe5c8f7b964d30e9027 18624cef6c6b679eeace2a98737e472432e0ead354cca02192b4d45330f047c9 lib/codeql/rust/elements/internal/generated/RangePat.qll efd93730de217cf50dcba5875595263a5eadf9f7e4e1272401342a094d158614 229b251b3d118932e31e78ac4dfb75f48b766f240f20d436062785606d44467b -lib/codeql/rust/elements/internal/generated/Raw.qll 7de290d66bd594f4c5b5a296502792e803e9f1084bb2616d9774196e33b16c87 28150fdd3cff3bb49b407f0c2119602be13e78cbb1f8fd749edd31f5d9772f7a +lib/codeql/rust/elements/internal/generated/Raw.qll c302331305186bdf4cfe2aa08395778be84d4959bb535353e1bc9979094e3de2 f40d71d96218cd6dc9b544dfe4aa2d6d035d21ca414ba1cf456741c90076e3db lib/codeql/rust/elements/internal/generated/RecordExpr.qll eb6cb662e463f9260efae1a6ce874fa781172063b916ef1963f861e9942d308d 1a21cbccc8f3799ff13281e822818ebfb21d81591720a427cac3625512cb9d40 lib/codeql/rust/elements/internal/generated/RecordExprField.qll 7e9f8663d3b74ebbc9603b10c9912f082febba6bd73d344b100bbd3edf837802 fbe6b578e7fd5d5a6f21bbb8c388957ab7210a6a249ec71510a50fb35b319ea1 lib/codeql/rust/elements/internal/generated/RecordExprFieldList.qll 179a97211fe7aa6265085d4d54115cdbc0e1cd7c9b2135591e8f36d6432f13d3 dd44bbbc1e83a1ed3a587afb729d7debf7aeb7b63245de181726af13090e50c0 @@ -551,8 +554,8 @@ lib/codeql/rust/elements/internal/generated/Static.qll 5fbd6879858cf356d4bdaa6da lib/codeql/rust/elements/internal/generated/Stmt.qll 8473ff532dd5cc9d7decaddcd174b94d610f6ca0aec8e473cc051dad9f3db917 6ef7d2b5237c2dbdcacbf7d8b39109d4dc100229f2b28b5c9e3e4fbf673ba72b lib/codeql/rust/elements/internal/generated/StmtList.qll a667193e32341e17400867c6e359878c4e645ef9f5f4d97676afc0283a33a026 a320ed678ee359302e2fc1b70a9476705cd616fcfa44a499d32f0c7715627f73 lib/codeql/rust/elements/internal/generated/Struct.qll 4d57f0db12dc7ad3e31e750a24172ef1505406b4dab16386af0674bd18bf8f4b 1a73c83df926b996f629316f74c61ea775be04532ab61b56af904223354f033e -lib/codeql/rust/elements/internal/generated/Synth.qll 65873a7fa44e223edc5e76cc768591a036eb2550960a6b6882476f43a01aefba 3e08e2bdfba53ae26d8f48f2d240b92b44c603f03105518c37a963e0cbe63e3f -lib/codeql/rust/elements/internal/generated/SynthConstructors.qll e929c49ea60810a2bbc19ad38110b8bbaf21db54dae90393b21a3459a54abf6f e929c49ea60810a2bbc19ad38110b8bbaf21db54dae90393b21a3459a54abf6f +lib/codeql/rust/elements/internal/generated/Synth.qll 678edaf70ea0fc2680e53c8cdde0114228be6d8935845f86ccec48274e32d82d 963abedc8cfa60b63db61f250e7da26fc886148408d5340ed500bad3e6dfd7b0 +lib/codeql/rust/elements/internal/generated/SynthConstructors.qll f33931fbdee7e1ca8b5e52af771ea725fef72a2e37fffb628f5270e79d3571a1 f33931fbdee7e1ca8b5e52af771ea725fef72a2e37fffb628f5270e79d3571a1 lib/codeql/rust/elements/internal/generated/Token.qll 77a91a25ca5669703cf3a4353b591cef4d72caa6b0b9db07bb9e005d69c848d1 2fdffc4882ed3a6ca9ac6d1fb5f1ac5a471ca703e2ffdc642885fa558d6e373b lib/codeql/rust/elements/internal/generated/TokenTree.qll 8577c2b097c1be2f0f7daa5acfcf146f78674a424d99563e08a84dd3e6d91b46 d2f30764e84dbfc0a6a5d3d8a5f935cd432413688cb32da9c94e420fbc10665c lib/codeql/rust/elements/internal/generated/Trait.qll 8fa41b50fa0f68333534f2b66bb4ec8e103ff09ac8fa5c2cc64bc04beafec205 ce1c9aa6d0e2f05d28aab8e1165c3b9fb8e24681ade0cf6a9df2e8617abeae7e @@ -586,7 +589,7 @@ lib/codeql/rust/elements/internal/generated/WhileExpr.qll 7edf1f23fbf953a2baabcd lib/codeql/rust/elements/internal/generated/WildcardPat.qll d74b70b57a0a66bfae017a329352a5b27a6b9e73dd5521d627f680e810c6c59e 4b913b548ba27ff3c82fcd32cf996ff329cb57d176d3bebd0fcef394486ea499 lib/codeql/rust/elements/internal/generated/YeetExpr.qll cac328200872a35337b4bcb15c851afb4743f82c080f9738d295571eb01d7392 94af734eea08129b587fed849b643e7572800e8330c0b57d727d41abda47930b lib/codeql/rust/elements/internal/generated/YieldExpr.qll 37e5f0c1e373a22bbc53d8b7f2c0e1f476e5be5080b8437c5e964f4e83fad79a 4a9a68643401637bf48e5c2b2f74a6bf0ddcb4ff76f6bffb61d436b685621e85 -lib/codeql/rust/elements.qll ced76fbeebc6e2e972ecaed65ef97851f90a215cf330f28a0f31a253f1c03442 ced76fbeebc6e2e972ecaed65ef97851f90a215cf330f28a0f31a253f1c03442 +lib/codeql/rust/elements.qll 34cd7db7d8c3cfeac77f23d07418df951158feea426df6ad7d80ebebbcb4e3f2 34cd7db7d8c3cfeac77f23d07418df951158feea426df6ad7d80ebebbcb4e3f2 test/extractor-tests/generated/Abi/Abi.ql 7f6e7dc4af86eca3ebdc79b10373988cd0871bd78b51997d3cffd969105e5fdd 2f936b6ca005c6157c755121584410c03e4a3949c23bee302fbe05ee10ce118f test/extractor-tests/generated/Abi/Abi_getAbiString.ql a496762fcec5a0887b87023bbf93e9b650f02e20113e25c44d6e4281ae8f5335 14109c7ce11ba25e3cd6e7f1b3fcb4cb00622f2a4eac91bfe43145c5f366bc52 test/extractor-tests/generated/ArgList/ArgList.ql e412927756e72165d0e7c5c9bd3fca89d08197bbf760db8fb7683c64bb2229bc 043dba8506946fbb87753e22c387987d7eded6ddb963aa067f9e60ef9024d684 diff --git a/rust/ql/.gitattributes b/rust/ql/.gitattributes index c75ea349c0d1..cca884a14268 100644 --- a/rust/ql/.gitattributes +++ b/rust/ql/.gitattributes @@ -224,6 +224,8 @@ /lib/codeql/rust/elements/internal/ExternItemImpl.qll linguist-generated /lib/codeql/rust/elements/internal/ExternItemListConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/ExternItemListImpl.qll linguist-generated +/lib/codeql/rust/elements/internal/ExtractorStep.qll linguist-generated +/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/FieldExprConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/FieldListImpl.qll linguist-generated /lib/codeql/rust/elements/internal/FnPtrTypeConstructor.qll linguist-generated @@ -451,6 +453,7 @@ /lib/codeql/rust/elements/internal/generated/ExternCrate.qll linguist-generated /lib/codeql/rust/elements/internal/generated/ExternItem.qll linguist-generated /lib/codeql/rust/elements/internal/generated/ExternItemList.qll linguist-generated +/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FieldExpr.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FieldList.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FnPtrType.qll linguist-generated diff --git a/rust/ql/integration-tests/hello-project/steps.ql b/rust/ql/integration-tests/hello-project/steps.ql index eb8f9a05adb7..17358a1c1000 100644 --- a/rust/ql/integration-tests/hello-project/steps.ql +++ b/rust/ql/integration-tests/hello-project/steps.ql @@ -1,4 +1,4 @@ -import codeql.rust.internal.ExtractorStep +import codeql.rust.elements.internal.ExtractorStep from ExtractorStep step select step diff --git a/rust/ql/integration-tests/hello-project/summary.expected b/rust/ql/integration-tests/hello-project/summary.expected index 09e03490eb13..44c14e790fe3 100644 --- a/rust/ql/integration-tests/hello-project/summary.expected +++ b/rust/ql/integration-tests/hello-project/summary.expected @@ -1,4 +1,4 @@ -| Elements extracted | 50 | +| Elements extracted | 65 | | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 1 | diff --git a/rust/ql/integration-tests/hello-workspace/steps.ql b/rust/ql/integration-tests/hello-workspace/steps.ql index eb8f9a05adb7..17358a1c1000 100644 --- a/rust/ql/integration-tests/hello-workspace/steps.ql +++ b/rust/ql/integration-tests/hello-workspace/steps.ql @@ -1,4 +1,4 @@ -import codeql.rust.internal.ExtractorStep +import codeql.rust.elements.internal.ExtractorStep from ExtractorStep step select step diff --git a/rust/ql/integration-tests/hello-workspace/summary.expected b/rust/ql/integration-tests/hello-workspace/summary.cargo.expected similarity index 94% rename from rust/ql/integration-tests/hello-workspace/summary.expected rename to rust/ql/integration-tests/hello-workspace/summary.cargo.expected index 4e800e60bd0c..ec1abea6252b 100644 --- a/rust/ql/integration-tests/hello-workspace/summary.expected +++ b/rust/ql/integration-tests/hello-workspace/summary.cargo.expected @@ -1,4 +1,4 @@ -| Elements extracted | 72 | +| Elements extracted | 86 | | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 0 | diff --git a/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected b/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected new file mode 100644 index 000000000000..5ca38e5a90ce --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected @@ -0,0 +1,17 @@ +| Elements extracted | 85 | +| Elements unextracted | 0 | +| Extraction errors | 0 | +| Extraction warnings | 0 | +| Files extracted - total | 4 | +| Files extracted - with errors | 0 | +| Files extracted - without errors | 4 | +| Inconsistencies - AST | 0 | +| Inconsistencies - CFG | 0 | +| Inconsistencies - data flow | 0 | +| Lines of code extracted | 9 | +| Lines of user code extracted | 9 | +| Macro calls - resolved | 2 | +| Macro calls - total | 2 | +| Macro calls - unresolved | 0 | +| Taint sources - active | 0 | +| Taint sources - total | 0 | diff --git a/rust/ql/integration-tests/hello-workspace/test_workspace.py b/rust/ql/integration-tests/hello-workspace/test_workspace.py index 7f9e3700c3a3..fe8dbc69141c 100644 --- a/rust/ql/integration-tests/hello-workspace/test_workspace.py +++ b/rust/ql/integration-tests/hello-workspace/test_workspace.py @@ -2,11 +2,13 @@ @pytest.mark.ql_test("steps.ql", expected=".cargo.expected") +@pytest.mark.ql_test("summary.qlref", expected=".cargo.expected") def test_cargo(codeql, rust, cargo, check_source_archive, rust_check_diagnostics): rust_check_diagnostics.expected_suffix = ".cargo.expected" codeql.database.create() @pytest.mark.ql_test("steps.ql", expected=".rust-project.expected") +@pytest.mark.ql_test("summary.qlref", expected=".rust-project.expected") def test_rust_project(codeql, rust, rust_project, check_source_archive, rust_check_diagnostics): rust_check_diagnostics.expected_suffix = ".rust-project.expected" codeql.database.create() diff --git a/rust/ql/lib/codeql/files/FileSystem.qll b/rust/ql/lib/codeql/files/FileSystem.qll index c83680f7ec69..b13793b36001 100644 --- a/rust/ql/lib/codeql/files/FileSystem.qll +++ b/rust/ql/lib/codeql/files/FileSystem.qll @@ -6,7 +6,7 @@ private import codeql.rust.elements.SourceFile private import codeql.rust.elements.AstNode private import codeql.rust.elements.Comment private import codeql.rust.Diagnostics -private import codeql.rust.internal.ExtractorStep +private import codeql.rust.elements.internal.ExtractorStep private module Input implements InputSig { abstract class ContainerBase extends @container { diff --git a/rust/ql/lib/codeql/rust/elements.qll b/rust/ql/lib/codeql/rust/elements.qll index e37dde90d618..7954e0e2b14f 100644 --- a/rust/ql/lib/codeql/rust/elements.qll +++ b/rust/ql/lib/codeql/rust/elements.qll @@ -3,6 +3,7 @@ * This module exports all modules providing `Element` subclasses. */ +import codeql.files.FileSystem import codeql.rust.elements.Abi import codeql.rust.elements.Addressable import codeql.rust.elements.ArgList diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll new file mode 100644 index 000000000000..64a4931ef43c --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll @@ -0,0 +1,13 @@ +// generated by codegen, do not edit +/** + * This module provides the class `ExtractorStep`. + */ + +private import ExtractorStepImpl +import codeql.rust.elements.Element +import codeql.files.FileSystem + +/** + * INTERNAL: Do not use. + */ +final class ExtractorStep = Impl::ExtractorStep; diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll new file mode 100644 index 000000000000..76050bf99f86 --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll @@ -0,0 +1,14 @@ +// generated by codegen, remove this comment if you wish to edit this file +/** + * This module defines the hook used internally to tweak the characteristic predicate of + * `ExtractorStep` synthesized instances. + * INTERNAL: Do not use. + */ + +private import codeql.rust.elements.internal.generated.Raw + +/** + * The characteristic predicate of `ExtractorStep` synthesized instances. + * INTERNAL: Do not use. + */ +predicate constructExtractorStep(Raw::ExtractorStep id) { any() } diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll new file mode 100644 index 000000000000..95c677e845e9 --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll @@ -0,0 +1,32 @@ +/** + * This module provides a hand-modifiable wrapper around the generated class `ExtractorStep`. + * + * INTERNAL: Do not use. + */ + +private import codeql.rust.elements.internal.generated.ExtractorStep + +/** + * INTERNAL: This module contains the customizable definition of `ExtractorStep` and should not + * be referenced directly. + */ +module Impl { + class ExtractorStep extends Generated::ExtractorStep { + override string toString() { + result = this.getAction() + "(" + this.getFile().getAbsolutePath() + ")" + } + + /** + * Provides location information for this step. + */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + this.getFile().getAbsolutePath() = filepath and + startline = 0 and + startcolumn = 0 and + endline = 0 and + endcolumn = 0 + } + } +} diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll new file mode 100644 index 000000000000..c2d7838865be --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll @@ -0,0 +1,45 @@ +// generated by codegen, do not edit +/** + * This module provides the generated definition of `ExtractorStep`. + * INTERNAL: Do not import directly. + */ + +private import codeql.rust.elements.internal.generated.Synth +private import codeql.rust.elements.internal.generated.Raw +import codeql.rust.elements.internal.ElementImpl::Impl as ElementImpl +import codeql.files.FileSystem + +/** + * INTERNAL: This module contains the fully generated definition of `ExtractorStep` and should not + * be referenced directly. + */ +module Generated { + /** + * INTERNAL: Do not reference the `Generated::ExtractorStep` class directly. + * Use the subclass `ExtractorStep`, where the following predicates are available. + */ + class ExtractorStep extends Synth::TExtractorStep, ElementImpl::Element { + override string getAPrimaryQlClass() { result = "ExtractorStep" } + + /** + * Gets the action of this extractor step. + */ + string getAction() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getAction() + } + + /** + * Gets the file of this extractor step. + */ + File getFile() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getFile() + } + + /** + * Gets the duration ms of this extractor step. + */ + int getDurationMs() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getDurationMs() + } + } +} diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll index 6709629e8b4f..0ba866f8ee29 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll @@ -4,12 +4,28 @@ */ import codeql.rust.elements +import codeql.rust.elements.internal.ExtractorStep private module Impl { private Element getImmediateChildOfElement(Element e, int index, string partialPredicateCall) { none() } + private Element getImmediateChildOfExtractorStep( + ExtractorStep e, int index, string partialPredicateCall + ) { + exists(int b, int bElement, int n | + b = 0 and + bElement = b + 1 + max(int i | i = -1 or exists(getImmediateChildOfElement(e, i, _)) | i) and + n = bElement and + ( + none() + or + result = getImmediateChildOfElement(e, index - b, partialPredicateCall) + ) + ) + } + private Element getImmediateChildOfLocatable(Locatable e, int index, string partialPredicateCall) { exists(int b, int bElement, int n | b = 0 and @@ -3600,6 +3616,8 @@ private module Impl { // * none() simplifies generation, as we can append `or ...` without a special case for the first item none() or + result = getImmediateChildOfExtractorStep(e, index, partialAccessor) + or result = getImmediateChildOfFormat(e, index, partialAccessor) or result = getImmediateChildOfFormatArgument(e, index, partialAccessor) diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll index 1c85daa22cfa..96db2c99cf6b 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll @@ -3,6 +3,8 @@ * This module holds thin fully generated class definitions around DB entities. */ module Raw { + private import codeql.files.FileSystem + /** * INTERNAL: Do not use. */ @@ -10,6 +12,28 @@ module Raw { string toString() { none() } } + /** + * INTERNAL: Do not use. + */ + class ExtractorStep extends @extractor_step, Element { + override string toString() { result = "ExtractorStep" } + + /** + * Gets the action of this extractor step. + */ + string getAction() { extractor_steps(this, result, _, _) } + + /** + * Gets the file of this extractor step. + */ + File getFile() { extractor_steps(this, _, result, _) } + + /** + * Gets the duration ms of this extractor step. + */ + int getDurationMs() { extractor_steps(this, _, _, result) } + } + /** * INTERNAL: Do not use. */ diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll index 0b4fa39cd848..5284c0b8fde8 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll @@ -134,6 +134,10 @@ module Synth { * INTERNAL: Do not use. */ TExternItemList(Raw::ExternItemList id) { constructExternItemList(id) } or + /** + * INTERNAL: Do not use. + */ + TExtractorStep(Raw::ExtractorStep id) { constructExtractorStep(id) } or /** * INTERNAL: Do not use. */ @@ -926,6 +930,12 @@ module Synth { */ TExternItemList convertExternItemListFromRaw(Raw::Element e) { result = TExternItemList(e) } + /** + * INTERNAL: Do not use. + * Converts a raw element to a synthesized `TExtractorStep`, if possible. + */ + TExtractorStep convertExtractorStepFromRaw(Raw::Element e) { result = TExtractorStep(e) } + /** * INTERNAL: Do not use. * Converts a raw element to a synthesized `TFieldExpr`, if possible. @@ -1803,6 +1813,8 @@ module Synth { * Converts a raw DB element to a synthesized `TElement`, if possible. */ TElement convertElementFromRaw(Raw::Element e) { + result = convertExtractorStepFromRaw(e) + or result = convertLocatableFromRaw(e) or result = convertUnextractedFromRaw(e) @@ -2312,6 +2324,12 @@ module Synth { */ Raw::Element convertExternItemListToRaw(TExternItemList e) { e = TExternItemList(result) } + /** + * INTERNAL: Do not use. + * Converts a synthesized `TExtractorStep` to a raw DB element, if possible. + */ + Raw::Element convertExtractorStepToRaw(TExtractorStep e) { e = TExtractorStep(result) } + /** * INTERNAL: Do not use. * Converts a synthesized `TFieldExpr` to a raw DB element, if possible. @@ -3187,6 +3205,8 @@ module Synth { * Converts a synthesized `TElement` to a raw DB element, if possible. */ Raw::Element convertElementToRaw(TElement e) { + result = convertExtractorStepToRaw(e) + or result = convertLocatableToRaw(e) or result = convertUnextractedToRaw(e) diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll index 9b68c858c6a5..8f7c2c615706 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll @@ -33,6 +33,7 @@ import codeql.rust.elements.internal.ExprStmtConstructor import codeql.rust.elements.internal.ExternBlockConstructor import codeql.rust.elements.internal.ExternCrateConstructor import codeql.rust.elements.internal.ExternItemListConstructor +import codeql.rust.elements.internal.ExtractorStepConstructor import codeql.rust.elements.internal.FieldExprConstructor import codeql.rust.elements.internal.FnPtrTypeConstructor import codeql.rust.elements.internal.ForExprConstructor diff --git a/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll b/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll deleted file mode 100644 index 50e6ebafebfe..000000000000 --- a/rust/ql/lib/codeql/rust/internal/ExtractorStep.qll +++ /dev/null @@ -1,46 +0,0 @@ -import codeql.files.FileSystem - -/** - * An extractor step, usable for debugging and diagnostics reasons. - * - * INTERNAL: Do not use. - */ -class ExtractorStep extends @extractor_step { - /** - * Gets the string representation of this extractor step. - */ - string toString() { - exists(File file, string action | - extractor_steps(this, action, file, _) and - result = action + "(" + file.getAbsolutePath() + ")" - ) - } - - /** - * Gets the action this extractor step carried out. - */ - string getAction() { extractor_steps(this, result, _, _) } - - /** - * Gets the file the extractor step was carried out on. - */ - File getFile() { extractor_steps(this, _, result, _) } - - /** - * Gets the duration of the extractor step in milliseconds. - */ - int getDurationMs() { extractor_steps(this, _, _, result) } - - /** - * Provides location information for this step. - */ - predicate hasLocationInfo( - string filepath, int startline, int startcolumn, int endline, int endcolumn - ) { - this.getFile().getAbsolutePath() = filepath and - startline = 0 and - startcolumn = 0 and - endline = 0 and - endcolumn = 0 - } -} diff --git a/rust/ql/lib/rust.dbscheme b/rust/ql/lib/rust.dbscheme index f0f2a65aea5f..9709ae9bf544 100644 --- a/rust/ql/lib/rust.dbscheme +++ b/rust/ql/lib/rust.dbscheme @@ -116,21 +116,22 @@ locatable_locations( int location: @location_default ref ); -extractor_steps( - unique int id: @extractor_step, - string action: string ref, - int file: @file ref, - int duration_ms: int ref -) - // from schema @element = - @locatable + @extractor_step +| @locatable | @unextracted ; +extractor_steps( + unique int id: @extractor_step, + string action: string ref, + int file: @file ref, + int duration_ms: int ref +); + @locatable = @ast_node ; diff --git a/rust/schema/prelude.py b/rust/schema/prelude.py index ffd65959b5ab..5b050872609b 100644 --- a/rust/schema/prelude.py +++ b/rust/schema/prelude.py @@ -3,6 +3,7 @@ include("../shared/tree-sitter-extractor/src/generator/prefix.dbscheme") include("prefix.dbscheme") +File = imported("File", "codeql.files.FileSystem") @qltest.skip class Element: @@ -93,3 +94,11 @@ class Resolvable(AstNode): """ resolved_path: optional[string] | rust.detach | ql.internal resolved_crate_origin: optional[string] | rust.detach | ql.internal + + +@qltest.skip +@ql.internal +class ExtractorStep(Element): + action: string + file: File + duration_ms: int diff --git a/swift/ql/.generated.list b/swift/ql/.generated.list index da39f2f2fe77..601205793210 100644 --- a/swift/ql/.generated.list +++ b/swift/ql/.generated.list @@ -712,7 +712,7 @@ lib/codeql/swift/generated/OtherAvailabilitySpec.qll d9feaa2a71acff3184ca389045b lib/codeql/swift/generated/ParentChild.qll d1814f2bad4c2ba9242ce49fe6fb8564ac99fc1fd3a7d12aa55e5c6dd7bb529b 1a2075b731d07a5e3c6a69d001796c8de925069d839671a294c9cba6c3db724a lib/codeql/swift/generated/PlatformVersionAvailabilitySpec.qll dc17b49a90a18a8f7607adf2433bc8f0c194fa3e803aa3822f809d4d4fbd6793 be48ea9f8ae17354c8508aaed24337a9e57ce01f288fece3dcecd99776cabcec lib/codeql/swift/generated/PureSynthConstructors.qll bc31a6c4d142fa3fbdcae69d5ba6f1cec00eb9ad92b46c8d7b91ebfa7ef6c1f4 bc31a6c4d142fa3fbdcae69d5ba6f1cec00eb9ad92b46c8d7b91ebfa7ef6c1f4 -lib/codeql/swift/generated/Raw.qll 118b43fedd4265b5aa15c33ef01a2f5a5db6e5597f95bef1078a01c3ff8da983 075aec2c8b232f0361ebf63f07ae9b66163f3975e6023583fb0fa2e40b979a33 +lib/codeql/swift/generated/Raw.qll a47950495630c10c00afee734d779e5e3e7f8e3e65ae3f04b9a1254e3e73921e 075aec2c8b232f0361ebf63f07ae9b66163f3975e6023583fb0fa2e40b979a33 lib/codeql/swift/generated/Synth.qll 31e318c6e156848c85a2a2664695b48b5e93c57c9bb22fa29d027069907b3ab0 8655ffcf772f55284b93f1d7f8e1b3d497a9744d5f2e0c17bc322c1fdf8bdba8 lib/codeql/swift/generated/SynthConstructors.qll 3e53c7853096020219c01dae85681fe80b34938d198a0ff359a209dda41c5ed7 3e53c7853096020219c01dae85681fe80b34938d198a0ff359a209dda41c5ed7 lib/codeql/swift/generated/UnknownFile.qll 247ddf2ebb49ce5ed4bf7bf91a969ddff37de6c78d43d8affccaf7eb586e06f2 452b29f0465ef45e978ef8b647b75e5a2a1e53f2a568fc003bc8f52f73b3fa4d