Resolve relative local dependencies of local dependencies

.changes/unreleased/Fixes-20240823-150343.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Resolve relative local dependencies of local dependencies
+time: 2024-08-23T15:03:43.292804-06:00
+custom:
+  Author: dbeatty10
+  Issue: "10600"

core/dbt/contracts/project.py

@@ -50,6 +50,7 @@ class Package(dbtClassMixin):
 class LocalPackage(Package):
     local: str
     unrendered: Dict[str, Any] = field(default_factory=dict)
+    project_root: Optional[str] = None
 
 
 # `float` also allows `int`, according to PEP484 (and jsonschema!)

core/dbt/deps/base.py

@@ -85,7 +85,7 @@ def nice_version_name(self):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def to_dict(self) -> Dict[str, str]:
+    def to_dict(self) -> Dict[str, Optional[str]]:
         raise NotImplementedError
 
     def fetch_metadata(self, project, renderer):

core/dbt/deps/git.py

@@ -58,11 +58,11 @@ def __init__(
         self.subdirectory = subdirectory
         self._checkout_name = md5sum(self.name)
 
-    def to_dict(self) -> Dict[str, str]:
+    def to_dict(self) -> Dict[str, Optional[str]]:
         git_scrubbed = scrub_secrets(self.git_unrendered, env_secrets())
         if self.git_unrendered != git_scrubbed:
             warn_or_error(DepsScrubbedPackageName(package_name=git_scrubbed))
-        ret = {
+        ret: Dict[str, Optional[str]] = {
             "git": git_scrubbed,
             "revision": self.revision,
         }

core/dbt/deps/local.py

@@ -1,5 +1,5 @@
 import shutil
-from typing import Dict
+from typing import Dict, Optional
 
 from dbt.config.project import PartialProject, Project
 from dbt.config.renderer import PackageRenderer
@@ -11,9 +11,10 @@
 
 
 class LocalPackageMixin:
-    def __init__(self, local: str) -> None:
+    def __init__(self, local: str, project_root: Optional[str] = None) -> None:
         super().__init__()
         self.local = local
+        self.project_root = project_root
 
     @property
     def name(self):
@@ -24,12 +25,13 @@
 
 
 class LocalPinnedPackage(LocalPackageMixin, PinnedPackage):
-    def __init__(self, local: str) -> None:
-        super().__init__(local)
+    def __init__(self, local: str, project_root: Optional[str] = None) -> None:
+        super().__init__(local, project_root)
 
-    def to_dict(self) -> Dict[str, str]:
+    def to_dict(self) -> Dict[str, Optional[str]]:
         return {
             "local": self.local,
+            "project_root": self.project_root,
         }
 
     def get_version(self):
@@ -38,10 +40,17 @@
     def nice_version_name(self):
         return "<local @ {}>".format(self.local)
 
-    def resolve_path(self, project):
+    def resolve_path(self, project: Project) -> str:
+        """If `self.local` is a relative path, create an absolute path
+        with either `self.project_root` or `project.project_root` as the base.
+
+        If `self.local` is an absolute path or a user path (~), just
+        resolve it to an absolute path and return.
+        """
+
         return system.resolve_path_from_base(
             self.local,
-            project.project_root,
+            self.project_root if self.project_root else project.project_root,
         )
 
     def _fetch_metadata(
@@ -70,10 +79,10 @@
 class LocalUnpinnedPackage(LocalPackageMixin, UnpinnedPackage[LocalPinnedPackage]):
     @classmethod
     def from_contract(cls, contract: LocalPackage) -> "LocalUnpinnedPackage":
-        return cls(local=contract.local)
+        return cls(local=contract.local, project_root=contract.project_root)
 
     def incorporate(self, other: "LocalUnpinnedPackage") -> "LocalUnpinnedPackage":
-        return LocalUnpinnedPackage(local=self.local)
+        return LocalUnpinnedPackage(local=other.local, project_root=other.project_root)
 
     def resolved(self) -> LocalPinnedPackage:
-        return LocalPinnedPackage(local=self.local)
+        return LocalPinnedPackage(local=self.local, project_root=self.project_root)

core/dbt/deps/registry.py

@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from dbt.clients import registry
 from dbt.contracts.project import RegistryPackage, RegistryPackageMetadata
@@ -37,7 +37,7 @@ def __init__(self, package: str, version: str, version_latest: str) -> None:
     def name(self):
         return self.package
 
-    def to_dict(self) -> Dict[str, str]:
+    def to_dict(self) -> Dict[str, Optional[str]]:
         return {
             "package": self.package,
             "version": self.version,

core/dbt/deps/resolver.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import Any, Dict, Iterator, List, NoReturn, Set, Type
+from typing import Any, Dict, Iterator, List, NoReturn, Optional, Set, Type
 
 from dbt.config import Project
 from dbt.config.renderer import PackageRenderer
@@ -66,11 +66,13 @@ def incorporate(self, package: UnpinnedPackage):
         else:
             self.packages[key] = package
 
-    def update_from(self, src: List[PackageSpec]) -> None:
+    def update_from(self, src: List[PackageSpec], project_root: Optional[str] = None) -> None:
         pkg: UnpinnedPackage
         for contract in src:
             if isinstance(contract, LocalPackage):
                 pkg = LocalUnpinnedPackage.from_contract(contract)
+                # Override the project root for the local package contract IFF it is provided
+                pkg.project_root = project_root if project_root else pkg.project_root
             elif isinstance(contract, TarballPackage):
                 pkg = TarballUnpinnedPackage.from_contract(contract)
             elif isinstance(contract, GitPackage):
@@ -86,9 +88,11 @@ def update_from(self, src: List[PackageSpec]) -> None:
             self.incorporate(pkg)
 
     @classmethod
-    def from_contracts(cls: Type["PackageListing"], src: List[PackageSpec]) -> "PackageListing":
+    def from_contracts(
+        cls: Type["PackageListing"], src: List[PackageSpec], project_root: Optional[str] = None
+    ) -> "PackageListing":
         self = cls({})
-        self.update_from(src)
+        self.update_from(src, project_root=project_root)
         return self
 
     def resolved(self) -> List[PinnedPackage]:
@@ -118,7 +122,7 @@ def resolve_packages(
     project: Project,
     cli_vars: Dict[str, Any],
 ) -> List[PinnedPackage]:
-    pending = PackageListing.from_contracts(packages)
+    pending = PackageListing.from_contracts(packages, project_root=project.project_root)
     final = PackageListing()
 
     renderer = PackageRenderer(cli_vars)
@@ -129,7 +133,14 @@ def resolve_packages(
         for package in pending:
             final.incorporate(package)
             target = final[package].resolved().fetch_metadata(project, renderer)
-            next_pending.update_from(target.packages)
+
+            # Hack to get the project root if it is a LocalPackage
+            # https://github.com/dbt-labs/dbt-core/issues/5410
+            project_root = None
+            if isinstance(package, LocalUnpinnedPackage):
+                project_root = package.resolved().resolve_path(project)
+
+            next_pending.update_from(target.packages, project_root=project_root)
         pending = next_pending
 
     resolved = final.resolved()

core/dbt/deps/tarball.py

@@ -1,7 +1,7 @@
 import functools
 import os
 from pathlib import Path
-from typing import Dict
+from typing import Dict, Optional
 
 from dbt.config.project import PartialProject
 from dbt.contracts.project import TarballPackage
@@ -39,7 +39,7 @@ def __init__(self, tarball: str, tarball_unrendered: str, package: str) -> None:
     def name(self):
         return self.package
 
-    def to_dict(self) -> Dict[str, str]:
+    def to_dict(self) -> Dict[str, Optional[str]]:
         tarball_scrubbed = scrub_secrets(self.tarball_unrendered, env_secrets())
         if self.tarball_unrendered != tarball_scrubbed:
             warn_or_error(DepsScrubbedPackageName(package_name=tarball_scrubbed))