From 70e26832c25ffc0e618bbc6f02aa2c59c9c71e6b Mon Sep 17 00:00:00 2001 From: Ralf Date: Mon, 25 Mar 2024 10:57:40 +0200 Subject: [PATCH] [HWORKS-936] Explicit model provenance backend (#1513) * [HWORKS-938] explicit provenance - model (#1725) * [HWORKS-938] explicit provenance - model * fix collector key bug * fix * cleanup * fixes * fixes * fix build * remove unnecessary imports from tests --------- Co-authored-by: Alexandru Ormenisan Co-authored-by: Alexandru Ormenisan Co-authored-by: bubriks * fix comunity * [HWORKS-936][APPEND] explicit provenance - model (#1743) * init * add expand users to models * mini fix --------- Co-authored-by: Alex Ormenisan Co-authored-by: Alexandru Ormenisan Co-authored-by: Alexandru Ormenisan --- .../api/modelregistry/models/ModelUtils.java | 106 ++++++++++ .../modelregistry/models/ModelsBuilder.java | 44 +++-- .../modelregistry/models/ModelsResource.java | 15 ++ .../modelregistry/models/dto/ModelDTO.java | 25 ++- .../FeatureGroupProvenanceResource.java | 3 +- .../FeatureViewProvenanceResource.java | 3 +- .../provenance/ModelProvenanceResource.java | 115 +++++++++++ .../TrainingDatasetProvenanceResource.java | 3 +- .../explicit/ProvExplicitLinksBuilder.java | 99 +++++++--- .../explicit/dto/ProvArtifactDTO.java | 4 +- .../explicit/ModelLinkController.java | 48 +++++ .../provenance/explicit/ModelLinkFacade.java | 37 ++++ .../provenance/explicit/ProvArtifact.java | 24 ++- .../explicit/ProvExplicitControllerIface.java | 14 ++ .../provenance/explicit/ProvExplicitLink.java | 6 +- .../entity/models/version/ModelVersion.java | 8 + .../entity/provenance/ModelLink.java | 182 ++++++++++++++++++ .../entity/provenance/ProvExplicitNode.java | 4 +- .../main/resources/META-INF/persistence.xml | 1 + 19 files changed, 684 insertions(+), 57 deletions(-) create mode 100644 hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ModelProvenanceResource.java create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ModelLinkController.java create mode 100644 hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ModelLinkFacade.java create mode 100644 hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/provenance/ModelLink.java diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelUtils.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelUtils.java index 12e72aae7d..904aff447b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelUtils.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelUtils.java @@ -18,30 +18,44 @@ import io.hops.hopsworks.api.modelregistry.models.dto.ModelDTO; import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.dataset.DatasetController; +import io.hops.hopsworks.common.featurestore.FeaturestoreFacade; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewFacade; +import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetFacade; import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; import io.hops.hopsworks.common.hdfs.DistributedFsService; import io.hops.hopsworks.common.hdfs.HdfsUsersController; import io.hops.hopsworks.common.hdfs.Utils; +import io.hops.hopsworks.common.provenance.explicit.ModelLinkController; import io.hops.hopsworks.common.util.AccessController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.persistence.entity.dataset.Dataset; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; +import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; import io.hops.hopsworks.persistence.entity.project.Project; +import io.hops.hopsworks.persistence.entity.provenance.ModelLink; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.restutils.RESTCodes; +import org.javatuples.Pair; import javax.ejb.EJB; import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import java.util.logging.Level; +import java.util.logging.Logger; @Stateless @TransactionAttribute(TransactionAttributeType.NEVER) public class ModelUtils { + private static final Logger LOGGER = Logger.getLogger(ModelUtils.class.getName()); @EJB private AccessController accessCtrl; @@ -53,6 +67,16 @@ public class ModelUtils { private HdfsUsersController hdfsUsersController; @EJB private DistributedFsService dfs; + @EJB + private FeaturestoreFacade fsFacade; + @EJB + private FeatureViewFacade fvFacade; + @EJB + private FeatureViewController fvCtrl; + @EJB + private TrainingDatasetFacade tdFacade; + @EJB + private ModelLinkController modelLinkCtrl; public String getModelsDatasetPath(Project userProject, Project modelRegistryProject) { String modelsPath = Utils.getProjectPath(userProject.getName()) + Settings.HOPS_MODELS_DATASET + "/"; @@ -126,9 +150,91 @@ public String getModelFullPath(Project modelRegistryProject, String modelName, I return Utils.getProjectPath(modelRegistryProject.getName()) + Settings.HOPS_MODELS_DATASET + "/" + modelName + "/" + modelVersion; } + + public String getModelFullPath(ModelVersion modelVersion) { + return getModelFullPath(modelVersion.getModel().getProject(), modelVersion.getModel().getName(), + modelVersion.getVersion()); + } public String[] getModelNameAndVersion(String mlId) { int splitIndex = mlId.lastIndexOf("_"); return new String[]{mlId.substring(0, splitIndex), mlId.substring(splitIndex + 1)}; } + + public ModelLink createExplicitProvenanceLink(ModelVersion model, ModelDTO modelDTO) { + FeatureView fv = null; + Integer tdVersion = null; + if(modelDTO.getTrainingDataset() != null) { + if(modelDTO.getTrainingDataset().getId() != null) { + TrainingDataset td = tdFacade.find(modelDTO.getTrainingDataset().getId()); + if(td == null) { + LOGGER.info("training dataset not found - cannot create model provenance link"); + } + return modelLinkCtrl.createParentLink(model, td); + } + Pair fvNameVersion = splitTdName(modelDTO.getTrainingDataset().getName()); + if(fvNameVersion == null) { + LOGGER.info("training dataset name is wrong - cannot create model provenance link"); + return null; + } + fv = getFeatureView(modelDTO.getTrainingDataset().getFeaturestoreId(), + fvNameVersion.getValue0(), fvNameVersion.getValue1()); + tdVersion = modelDTO.getTrainingDataset().getVersion(); + } else if(modelDTO.getFeatureView() != null) { + if(modelDTO.getFeatureView().getId() != null) { + fv = fvFacade.find(modelDTO.getFeatureView().getId()); + } else { + fv = getFeatureView(modelDTO.getFeatureView().getFeaturestoreId(), + modelDTO.getFeatureView().getName(), modelDTO.getFeatureView().getVersion()); + } + tdVersion = modelDTO.getTrainingDatasetVersion(); + } + if(fv == null) { + return null; + } + if(tdVersion == null) { + LOGGER.info("training dataset version is missing - cannot create model provenance link"); + return null; + } + try { + TrainingDataset trainingDataset = tdFacade.findByFeatureViewAndVersion(fv, modelDTO.getTrainingDatasetVersion()); + return modelLinkCtrl.createParentLink(model, trainingDataset); + } catch (FeaturestoreException e) { + LOGGER.info("training dataset not found - cannot create model provenance link"); + return null; + } + } + + private Pair splitTdName(String tdName) { + int split = tdName.lastIndexOf("_"); + if(split == -1) { + return null; + } + try { + String fvName = tdName.substring(0, split); + int fvVersion; + try { + fvVersion = Integer.parseInt(tdName.substring(split + 1)); + } catch (NumberFormatException e) { + return null; + } + return Pair.with(fvName, fvVersion); + } catch (IndexOutOfBoundsException e) { + return null; + } + } + + private FeatureView getFeatureView(Integer fsId, String fvName, Integer fvVersion) { + Featurestore fvFS = fsFacade.findById(fsId); + if(fvFS == null) { + LOGGER.info("feature view featurestore not found - cannot create model provenance link"); + return null; + } + try { + return fvCtrl.getByNameVersionAndFeatureStore(fvName, fvVersion, fvFS); + } catch (FeaturestoreException e) { + LOGGER.info("feature view not found - cannot create model provenance link"); + return null; + } + } } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBuilder.java index 537c24fa92..16307c98ea 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsBuilder.java @@ -43,6 +43,7 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; +import javax.ws.rs.core.UriBuilder; import javax.ws.rs.core.UriInfo; import java.util.logging.Level; import java.util.logging.Logger; @@ -68,29 +69,31 @@ public class ModelsBuilder { private UsersBuilder usersBuilder; public ModelDTO uri(ModelDTO dto, UriInfo uriInfo, Project userProject, Project modelRegistryProject) { - dto.setHref(uriInfo.getBaseUriBuilder() - .path(ResourceRequest.Name.PROJECT.toString().toLowerCase()) - .path(Integer.toString(userProject.getId())) - .path(ResourceRequest.Name.MODELREGISTRIES.toString().toLowerCase()) - .path(Integer.toString(modelRegistryProject.getId())) - .path(ResourceRequest.Name.MODELS.toString().toLowerCase()) - .build()); + dto.setHref(modelUri(uriInfo, userProject, modelRegistryProject).build()); return dto; } public ModelDTO uri(ModelDTO dto, UriInfo uriInfo, Project userProject, Project modelRegistryProject, ModelVersion modelVersion) { - dto.setHref(uriInfo.getBaseUriBuilder() - .path(ResourceRequest.Name.PROJECT.toString().toLowerCase()) - .path(Integer.toString(userProject.getId())) - .path(ResourceRequest.Name.MODELREGISTRIES.toString().toLowerCase()) - .path(Integer.toString(modelRegistryProject.getId())) - .path(ResourceRequest.Name.MODELS.toString().toLowerCase()) - .path(modelVersion.getModel().getName() + "_" + modelVersion.getVersion()) - .build()); + dto.setHref(modelVersionUri(uriInfo, userProject, modelRegistryProject, modelVersion).build()); return dto; } + public UriBuilder modelUri(UriInfo uriInfo, Project userProject, Project modelRegistryProject) { + return uriInfo.getBaseUriBuilder() + .path(ResourceRequest.Name.PROJECT.toString().toLowerCase()) + .path(Integer.toString(userProject.getId())) + .path(ResourceRequest.Name.MODELREGISTRIES.toString().toLowerCase()) + .path(Integer.toString(modelRegistryProject.getId())) + .path(ResourceRequest.Name.MODELS.toString().toLowerCase()); + } + + public UriBuilder modelVersionUri(UriInfo uriInfo, Project userProject, Project modelRegistryProject, + ModelVersion modelVersion) { + return modelUri(uriInfo, userProject, modelRegistryProject) + .path(modelVersion.getModel().getName() + "_" + modelVersion.getVersion()); + } + public ModelDTO expand(ModelDTO dto, ResourceRequest resourceRequest) { if (resourceRequest != null && resourceRequest.contains(ResourceRequest.Name.MODELS)) { dto.setExpand(true); @@ -195,4 +198,15 @@ public ModelDTO build(UriInfo uriInfo, } return modelDTO; } + + public ModelDTO build(UriInfo uriInfo, + ResourceRequest resourceRequest, + Users user, + Project userProject, + ModelVersion modelVersion) + throws GenericException, ModelRegistryException, MetadataException, FeatureStoreMetadataException, + DatasetException { + return build(uriInfo, resourceRequest, user, userProject, modelVersion.getModel().getProject(), modelVersion, + modelUtils.getModelFullPath(modelVersion)); + } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsResource.java index bb9945223b..a5ea9a4f7c 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsResource.java @@ -23,6 +23,7 @@ import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.modelregistry.models.dto.ModelDTO; import io.hops.hopsworks.api.modelregistry.models.tags.ModelTagResource; +import io.hops.hopsworks.api.provenance.ModelProvenanceResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.hdfs.DistributedFsService; @@ -90,6 +91,8 @@ public class ModelsResource { private ModelUtils modelUtils; @Inject private ModelTagResource tagResource; + @Inject + private ModelProvenanceResource provenanceResource; private Project userProject; @@ -229,6 +232,7 @@ public Response put(@PathParam("id") String id, accessor = modelUtils.getModelsAccessor(user, userProject, modelProject, experimentProject); ModelVersion modelVersion = modelsController.createModelVersion(accessor, modelDTO, jobName, kernelId); + modelUtils.createExplicitProvenanceLink(modelVersion, modelDTO); ModelDTO dto = modelsBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.MODELS), user, userProject, modelRegistryProject, modelVersion, modelUtils.getModelFullPath(modelProject, modelVersion.getModel().getName(), modelVersion.getVersion())); @@ -251,4 +255,15 @@ public ModelTagResource tags(@ApiParam(value = "Id of the model", required = tru this.tagResource.setModel(modelVersion); return this.tagResource; } + + @Path("/{id}/provenance") + public ModelProvenanceResource provenance(@ApiParam(value = "Id of the model", required = true) + @PathParam("id") String id) + throws ModelRegistryException, ProvenanceException { + this.provenanceResource.setAccessProject(userProject); + this.provenanceResource.setModelRegistry(modelRegistryProject); + ModelVersion modelVersion = modelsController.getModel(modelRegistryProject, id); + this.provenanceResource.setModelVersion(modelVersion); + return this.provenanceResource; + } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/dto/ModelDTO.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/dto/ModelDTO.java index 513a580f27..48410cef4b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/dto/ModelDTO.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/dto/ModelDTO.java @@ -18,6 +18,7 @@ import io.hops.hopsworks.api.dataset.inode.InodeDTO; import io.hops.hopsworks.api.user.UserDTO; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewDTO; import io.hops.hopsworks.common.tags.TagsDTO; import io.hops.hopsworks.common.api.RestDTO; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO; @@ -78,6 +79,10 @@ public ModelDTO() { private Integer modelRegistryId; private TagsDTO tags; + + private FeatureViewDTO featureView; + + private int trainingDatasetVersion; private String type = "modelDTO"; @@ -234,7 +239,23 @@ public UserDTO getCreator() { public void setCreator(UserDTO creator) { this.creator = creator; } - + + public FeatureViewDTO getFeatureView() { + return featureView; + } + + public void setFeatureView(FeatureViewDTO featureView) { + this.featureView = featureView; + } + + public int getTrainingDatasetVersion() { + return trainingDatasetVersion; + } + + public void setTrainingDatasetVersion(int trainingDatasetVersion) { + this.trainingDatasetVersion = trainingDatasetVersion; + } + @Override public String toString() { return "ModelDTO{" + @@ -256,6 +277,8 @@ public String toString() { ", trainingDataset=" + trainingDataset + ", modelRegistryId=" + modelRegistryId + ", tags=" + tags + + ", featureView=" + featureView + + ", trainingDatasetVersion=" + trainingDatasetVersion + '}'; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureGroupProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureGroupProvenanceResource.java index 8bdda0579e..46328c88e6 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureGroupProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureGroupProvenanceResource.java @@ -42,6 +42,7 @@ import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; +import io.hops.hopsworks.exceptions.ModelRegistryException; import io.hops.hopsworks.exceptions.ProvenanceException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; @@ -138,7 +139,7 @@ public Response getLinks( @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException { + FeatureStoreMetadataException, IOException, ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureViewProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureViewProvenanceResource.java index 151894a0f0..14e6fb1436 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureViewProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureViewProvenanceResource.java @@ -38,6 +38,7 @@ import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; +import io.hops.hopsworks.exceptions.ModelRegistryException; import io.hops.hopsworks.exceptions.ProvenanceException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; @@ -132,7 +133,7 @@ public Response getLinks( @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException { + FeatureStoreMetadataException, IOException, ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ModelProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ModelProvenanceResource.java new file mode 100644 index 0000000000..ad284381de --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ModelProvenanceResource.java @@ -0,0 +1,115 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.provenance; + +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.filter.AllowedProjectRoles; +import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.provenance.explicit.ExplicitProvenanceExpansionBeanParam; +import io.hops.hopsworks.api.provenance.explicit.ProvExplicitLinksBuilder; +import io.hops.hopsworks.api.provenance.explicit.dto.ProvExplicitLinkDTO; +import io.hops.hopsworks.api.provenance.ops.ProvLinksBeanParams; +import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.provenance.explicit.ProvExplicitControllerIface; +import io.hops.hopsworks.common.provenance.explicit.ProvExplicitLink; +import io.hops.hopsworks.common.provenance.ops.dto.ProvLinksDTO; +import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.GenericException; +import io.hops.hopsworks.exceptions.MetadataException; +import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ServiceException; +import io.hops.hopsworks.jwt.annotation.JWTRequired; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; +import io.hops.hopsworks.persistence.entity.project.Project; +import io.hops.hopsworks.persistence.entity.user.Users; +import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; + +import javax.ejb.EJB; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; +import javax.enterprise.context.RequestScoped; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.BeanParam; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; +import java.io.IOException; + +@RequestScoped +@TransactionAttribute(TransactionAttributeType.NEVER) +@Api(value = "Model Explicit Provenance Resource") +public class ModelProvenanceResource { + @EJB + private JWTHelper jwtHelper; + @Inject + private ProvExplicitControllerIface provCtrl; + @EJB + private ProvExplicitLinksBuilder linksBuilder; + private Project accessProject; + private Project modelRegistry; + private ModelVersion modelVersion; + + public void setAccessProject(Project project) { + this.accessProject = project; + } + + public void setModelRegistry(Project project) { + this.modelRegistry = project; + } + + public void setModelVersion(ModelVersion modelVersion){ + this.modelVersion = modelVersion; + } + + @GET + @Path("links") + @Produces(MediaType.APPLICATION_JSON) + @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER}) + @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, + allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) + @ApiKeyRequired(acceptedScopes = {ApiScope.MODELREGISTRY}, + allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) + @ApiOperation(value = "Links Provenance query endpoint", + response = ProvLinksDTO.class) + public Response getLinks( + @BeanParam ProvLinksBeanParams params, + @BeanParam LinksPagination pagination, + @BeanParam ExplicitProvenanceExpansionBeanParam explicitProvenanceExpansionBeanParam, + @Context UriInfo uriInfo, + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, + FeatureStoreMetadataException, IOException, ModelRegistryException { + Users user = jwtHelper.getUserPrincipal(sc); + ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); + resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); + ProvExplicitLink provenance = provCtrl.modelLinks(accessProject, modelVersion, + pagination.getUpstreamLvls(), pagination.getDownstreamLvls()); + ProvExplicitLinkDTO result = linksBuilder.build(uriInfo, resourceRequest, accessProject, user, provenance); + return Response.ok().entity(result).build(); + } +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/TrainingDatasetProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/TrainingDatasetProvenanceResource.java index 59f1589462..f3181cbb41 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/TrainingDatasetProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/TrainingDatasetProvenanceResource.java @@ -38,6 +38,7 @@ import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; +import io.hops.hopsworks.exceptions.ModelRegistryException; import io.hops.hopsworks.exceptions.ProvenanceException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; @@ -126,7 +127,7 @@ public Response getLinks( @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException { + FeatureStoreMetadataException, IOException, ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/explicit/ProvExplicitLinksBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/explicit/ProvExplicitLinksBuilder.java index 6a6ad721db..068c0217e4 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/explicit/ProvExplicitLinksBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/explicit/ProvExplicitLinksBuilder.java @@ -17,6 +17,8 @@ import io.hops.hopsworks.api.featurestore.featureview.FeatureViewBuilder; import io.hops.hopsworks.api.featurestore.trainingdataset.TrainingDatasetDTOBuilder; +import io.hops.hopsworks.api.modelregistry.models.ModelsBuilder; +import io.hops.hopsworks.api.modelregistry.models.dto.ModelDTO; import io.hops.hopsworks.api.provenance.explicit.dto.ProvArtifactDTO; import io.hops.hopsworks.api.provenance.explicit.dto.featurestore.ProvCachedFeatureGroupDTO; import io.hops.hopsworks.api.provenance.explicit.dto.featurestore.ProvOnDemandFeatureGroupDTO; @@ -36,14 +38,16 @@ import io.hops.hopsworks.api.provenance.explicit.dto.ProvNodeDTO; import io.hops.hopsworks.api.provenance.explicit.dto.ProvExplicitLinkDTO; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; -import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; +import io.hops.hopsworks.exceptions.ModelRegistryException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.restutils.RESTCodes; @@ -72,7 +76,9 @@ public class ProvExplicitLinksBuilder { private FeatureViewBuilder featureViewBuilder; @EJB private TrainingDatasetDTOBuilder trainingDatasetBuilder; - + @EJB + private ModelsBuilder modelsBuilder; + public ProvExplicitLinksBuilder() {} //test @@ -103,34 +109,40 @@ public UriBuilder trainingDatasetURI(UriInfo uriInfo, Project accessProject, Tra .path(ResourceRequest.Name.PROVENANCE.toString().toLowerCase()) .path(ResourceRequest.Name.LINKS.toString().toLowerCase()); } + + public UriBuilder modelURI(UriInfo uriInfo, Project accessProject, ModelVersion model) { + return modelsBuilder.modelVersionUri(uriInfo, accessProject, model.getModel().getProject(), model) + .path(ResourceRequest.Name.PROVENANCE.toString().toLowerCase()) + .path(ResourceRequest.Name.LINKS.toString().toLowerCase()); + } public ProvExplicitLinkDTO buildFGLinks(UriInfo uriInfo, ResourceRequest resourceRequest, Project accessProject, Users user, ProvExplicitLink links) - throws DatasetException, FeatureStoreMetadataException, MetadataException, ServiceException, - GenericException, FeaturestoreException, IOException { + throws DatasetException, FeatureStoreMetadataException, MetadataException, ServiceException, + GenericException, FeaturestoreException, IOException, ModelRegistryException { return (ProvExplicitLinkDTO)build(uriInfo, resourceRequest, accessProject, user, links); } public ProvExplicitLinkDTO buildFVLinks(UriInfo uriInfo, ResourceRequest resourceRequest, Project accessProject, Users user, ProvExplicitLink links) - throws DatasetException, FeatureStoreMetadataException, MetadataException, ServiceException, - GenericException, FeaturestoreException, IOException { + throws DatasetException, FeatureStoreMetadataException, MetadataException, ServiceException, + GenericException, FeaturestoreException, IOException, ModelRegistryException { return (ProvExplicitLinkDTO)build(uriInfo, resourceRequest, accessProject, user, links); } public ProvExplicitLinkDTO buildTDLinks(UriInfo uriInfo, ResourceRequest resourceRequest, Project accessProject, Users user, ProvExplicitLink links) - throws DatasetException, FeatureStoreMetadataException, MetadataException, ServiceException, - GenericException, FeaturestoreException, IOException { + throws DatasetException, FeatureStoreMetadataException, MetadataException, ServiceException, + GenericException, FeaturestoreException, IOException, ModelRegistryException { return (ProvExplicitLinkDTO)build(uriInfo, resourceRequest, accessProject, user, links); } public ProvExplicitLinkDTO build(UriInfo uriInfo, ResourceRequest resourceRequest, Project accessProject, Users user, ProvExplicitLink links) - throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException { + throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, + FeatureStoreMetadataException, IOException, ModelRegistryException { boolean expandLink = resourceRequest != null && resourceRequest.contains(ResourceRequest.Name.PROVENANCE); boolean expandArtifact = resourceRequest != null && resourceRequest.contains(ResourceRequest.Name.PROVENANCE_ARTIFACTS); @@ -161,14 +173,23 @@ public ProvExplicitLinkDTO build(UriInfo uriInfo, ResourceRequest resourceReq linksDTO.setHref(trainingDatasetURI(uriInfo, accessProject, trainingDataset).build()); return linksDTO; } + } else if (links.getNode() instanceof ModelVersion) { + if(expandLink) { + return modelLink(uriInfo, accessProject, user, expandArtifact, links); + } else { + ProvExplicitLinkDTO linksDTO = new ProvExplicitLinkDTO<>(); + ModelVersion model = (ModelVersion) links.getNode(); + linksDTO.setHref(modelURI(uriInfo, accessProject, model).build()); + return linksDTO; + } } return null; } private ProvExplicitLinkDTO featureGroupLink(UriInfo uriInfo, Project accessProject, Users user, boolean expandArtifact, ProvExplicitLink links) - throws FeaturestoreException, ServiceException, MetadataException, FeatureStoreMetadataException, DatasetException, - IOException { + throws FeaturestoreException, ServiceException, MetadataException, FeatureStoreMetadataException, + DatasetException, IOException, GenericException, ModelRegistryException { ProvExplicitLinkDTO linksDTO = new ProvExplicitLinkDTO<>(); RestDTO artifactDTO; if(links.isDeleted()) { @@ -196,8 +217,8 @@ private ProvExplicitLinkDTO featureGroupLink(UriInfo uriInfo, Project accessProj private ProvExplicitLinkDTO featureViewLink(UriInfo uriInfo, Project accessProject, Users user, boolean expandArtifact, ProvExplicitLink links) - throws FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException { + throws FeaturestoreException, DatasetException, ServiceException, MetadataException, + FeatureStoreMetadataException, IOException, GenericException, ModelRegistryException { ProvExplicitLinkDTO linksDTO = new ProvExplicitLinkDTO<>(); RestDTO artifactDTO; if(links.isDeleted()) { @@ -218,7 +239,7 @@ private ProvExplicitLinkDTO featureViewLink(UriInfo uriInfo, Project accessProje linksDTO.setNode(buildNodeDTO(links, artifactDTO)); } catch (FeaturestoreException e) { if (e.getErrorCode().equals(RESTCodes.FeaturestoreErrorCode.FEATUREGROUP_NOT_FOUND)) { - artifactDTO = new ProvArtifactDTO(featureView.getId().toString(), + artifactDTO = new ProvArtifactDTO(featureView.getId(), featureView.getFeaturestore().getProject().getName(), featureView.getName(), featureView.getVersion()); linksDTO.setNode(buildNodeDTO(links, artifactDTO, Optional.of(e))); @@ -241,8 +262,8 @@ private ProvExplicitLinkDTO featureViewLink(UriInfo uriInfo, Project accessProje private ProvExplicitLinkDTO trainingDatasetLink(UriInfo uriInfo, Project accessProject, Users user, boolean expandArtifact, ProvExplicitLink links) - throws FeaturestoreException, DatasetException, ServiceException, MetadataException, FeatureStoreMetadataException, - IOException { + throws FeaturestoreException, DatasetException, ServiceException, MetadataException, + FeatureStoreMetadataException, IOException, GenericException, ModelRegistryException { ProvExplicitLinkDTO linksDTO = new ProvExplicitLinkDTO<>(); RestDTO artifactDTO; @@ -269,11 +290,44 @@ private ProvExplicitLinkDTO trainingDatasetLink(UriInfo uriInfo, Project accessP traverseLinks(uriInfo, accessProject, user, linksDTO, expandArtifact, links); return linksDTO; } + + private ProvExplicitLinkDTO modelLink(UriInfo uriInfo, Project accessProject, Users user, + boolean expandArtifact, ProvExplicitLink links) + throws FeaturestoreException, DatasetException, ServiceException, MetadataException, + FeatureStoreMetadataException, IOException, GenericException, ModelRegistryException { + ProvExplicitLinkDTO linksDTO = new ProvExplicitLinkDTO<>(); + RestDTO artifactDTO; + + if (links.isDeleted()) { + ProvArtifact artifact = (ProvArtifact) links.getNode(); + artifactDTO = new ProvArtifactDTO(artifact.getId(), artifact.getProject(), + artifact.getName(), artifact.getVersion()); + } else { + ModelVersion model = (ModelVersion) links.getNode(); + linksDTO.setHref(modelURI(uriInfo, accessProject, model).build()); + if (links.isAccessible() && expandArtifact) { + ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.MODELS); + resourceRequest.setExpansions(new HashSet<>(Arrays.asList( + new ResourceRequest(ResourceRequest.Name.USERS)))); + artifactDTO = modelsBuilder.build(uriInfo, resourceRequest, user, accessProject, model); + } else { + ProvArtifact artifact = ProvArtifact.fromModel(model); + artifactDTO = new ProvArtifactDTO(artifact.getId(), artifact.getProject(), + artifact.getName(), artifact.getVersion()); + } + artifactDTO.setHref( + modelsBuilder.modelVersionUri(uriInfo, accessProject, model.getModel().getProject(), model) + .queryParam("expand", "USERS").build()); + } + linksDTO.setNode(buildNodeDTO(links, artifactDTO)); + traverseLinks(uriInfo, accessProject, user, linksDTO, expandArtifact, links); + return linksDTO; + } private void traverseLinks(UriInfo uriInfo, Project accessProject, Users user, ProvExplicitLinkDTO linksDTO, boolean expandArtifact, ProvExplicitLink links) - throws FeaturestoreException, ServiceException, MetadataException, FeatureStoreMetadataException, DatasetException, - IOException { + throws FeaturestoreException, ServiceException, MetadataException, FeatureStoreMetadataException, + DatasetException, IOException, GenericException, ModelRegistryException { if(linksDTO.getNode().isAccessible()) { for (ProvExplicitLink downstreamLink : links.getDownstream()) { ProvExplicitLinkDTO downstreamLinkDTO = @@ -290,16 +344,17 @@ private void traverseLinks(UriInfo uriInfo, Project accessProject, Users user, P private ProvExplicitLinkDTO traverseLinksInt(UriInfo uriInfo, Project accessProject, Users user, boolean expandArtifact, ProvExplicitLink link) - throws FeaturestoreException, ServiceException, DatasetException, MetadataException, FeatureStoreMetadataException, - IOException { + throws FeaturestoreException, ServiceException, DatasetException, MetadataException, + FeatureStoreMetadataException, IOException, GenericException, ModelRegistryException { switch(link.getArtifactType()) { case FEATURE_GROUP: - case EXTERNAL_FEATURE_GROUP: return featureGroupLink(uriInfo, accessProject, user, expandArtifact, link); case FEATURE_VIEW: return featureViewLink(uriInfo, accessProject, user, expandArtifact, link); case TRAINING_DATASET: return trainingDatasetLink(uriInfo, accessProject, user, expandArtifact, link); + case MODEL: + return modelLink(uriInfo, accessProject, user, expandArtifact, link); default: return null; } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/explicit/dto/ProvArtifactDTO.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/explicit/dto/ProvArtifactDTO.java index c3b5c4b966..7aa3a6c763 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/explicit/dto/ProvArtifactDTO.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/explicit/dto/ProvArtifactDTO.java @@ -31,12 +31,12 @@ public ProvArtifactDTO() { super(); } - public ProvArtifactDTO(String id, String project, String name, Integer version) { + public ProvArtifactDTO(Integer id, String project, String name, Integer version) { super(); this.project = project; this.name = name; this.version = version; - this.id = id; + this.id = String.valueOf(id); } public String getId() { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ModelLinkController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ModelLinkController.java new file mode 100644 index 0000000000..e52383a21c --- /dev/null +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ModelLinkController.java @@ -0,0 +1,48 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.common.provenance.explicit; + +import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; +import io.hops.hopsworks.persistence.entity.provenance.ModelLink; + +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; + +/** + * Class controlling the interaction with the model_link table and required business logic + */ +@Stateless +@TransactionAttribute(TransactionAttributeType.NEVER) +public class ModelLinkController { + @EJB + private ModelLinkFacade modelLinkFacade; + + public ModelLink createParentLink(ModelVersion model, TrainingDataset trainingDataset) { + ModelLink link = new ModelLink(); + link.setModel(model); + link.setParentTrainingDataset(trainingDataset); + //we denormalize here so that if the parent gets deleted, there is still user readable information left + link.setParentFeatureStore(trainingDataset.getFeaturestore().getProject().getName()); + link.setParentFeatureViewName(trainingDataset.getFeatureView().getName()); + link.setParentFeatureViewVersion(trainingDataset.getFeatureView().getVersion()); + link.setParentTrainingDatasetVersion(trainingDataset.getVersion()); + modelLinkFacade.persist(link); + return link; + } +} \ No newline at end of file diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ModelLinkFacade.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ModelLinkFacade.java new file mode 100644 index 0000000000..5fbc2eba72 --- /dev/null +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ModelLinkFacade.java @@ -0,0 +1,37 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.common.provenance.explicit; + +import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; +import io.hops.hopsworks.persistence.entity.provenance.ModelLink; + +import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; + +/** + * A facade for the model_link table in the Hopsworks database, + * use this interface when performing database operations against the table. + */ +@Stateless +@TransactionAttribute(TransactionAttributeType.REQUIRED) +public class ModelLinkFacade extends LinkFacade { + + public ModelLinkFacade() { + super(ModelLinkFacade.class, ModelLink.class); + } +} \ No newline at end of file diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvArtifact.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvArtifact.java index 287a86d859..890dae528c 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvArtifact.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvArtifact.java @@ -18,11 +18,12 @@ import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; import io.hops.hopsworks.persistence.entity.provenance.ProvExplicitNode; public class ProvArtifact { - private String id; + private Integer id; private String project; private String name; private Integer version; @@ -30,32 +31,37 @@ public class ProvArtifact { public ProvArtifact() { } - public ProvArtifact(String artifactId, String projectName, String artifactName, Integer artifactVersion) { + public ProvArtifact(Integer id, String projectName, String artifactName, Integer artifactVersion) { this.project = projectName; this.name = artifactName; this.version = artifactVersion; - this.id = artifactId; + this.id = id; } public static ProvArtifact fromLinkAsParent(ProvExplicitNode link) { - return new ProvArtifact(String.valueOf(link.parentId()), link.parentProject(), + return new ProvArtifact(link.parentId(), link.parentProject(), link.parentName(), link.parentVersion()); } public static ProvArtifact fromFeatureGroup(Featuregroup fg) { - return new ProvArtifact(String.valueOf(fg.getId()), fg.getFeaturestore().getProject().getName(), + return new ProvArtifact(fg.getId(), fg.getFeaturestore().getProject().getName(), fg.getName(), fg.getVersion()); } public static ProvArtifact fromFeatureView(FeatureView fv) { - return new ProvArtifact(String.valueOf(fv.getId()), fv.getFeaturestore().getProject().getName(), + return new ProvArtifact(fv.getId(), fv.getFeaturestore().getProject().getName(), fv.getName(), fv.getVersion()); } public static ProvArtifact fromTrainingDataset(TrainingDataset td) { - return new ProvArtifact(String.valueOf(td.getId()), td.getFeaturestore().getProject().getName(), + return new ProvArtifact(td.getId(), td.getFeaturestore().getProject().getName(), td.getName(), td.getVersion()); } + + public static ProvArtifact fromModel(ModelVersion model) { + return new ProvArtifact(model.getId(), model.getModel().getProject().getName(), + model.getModel().getName(), model.getVersion()); + } public String getProject() { return project; @@ -81,11 +87,11 @@ public void setVersion(Integer version) { this.version = version; } - public String getId() { + public Integer getId() { return id; } - public void setId(String id) { + public void setId(Integer id) { this.id = id; } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvExplicitControllerIface.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvExplicitControllerIface.java index cee420efe2..b0ea7ad824 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvExplicitControllerIface.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvExplicitControllerIface.java @@ -21,6 +21,7 @@ import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.restutils.RESTCodes; @@ -65,4 +66,17 @@ default ProvExplicitLink trainingDatasetLinks(Project accessPro throws GenericException, FeaturestoreException, DatasetException { throw new GenericException(RESTCodes.GenericErrorCode.ENTERPRISE_FEATURE, Level.FINE); } + + default ProvExplicitLink modelLinks(Project accessProject, ModelVersion root) + throws GenericException, FeaturestoreException, DatasetException { + throw new GenericException(RESTCodes.GenericErrorCode.ENTERPRISE_FEATURE, Level.FINE); + } + + default ProvExplicitLink modelLinks(Project accessProject, + ModelVersion root, + Integer upstreamLevels, + Integer downstreamLevels) + throws GenericException, FeaturestoreException, DatasetException { + throw new GenericException(RESTCodes.GenericErrorCode.ENTERPRISE_FEATURE, Level.FINE); + } } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvExplicitLink.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvExplicitLink.java index f4b516b120..de59601bef 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvExplicitLink.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/provenance/explicit/ProvExplicitLink.java @@ -30,7 +30,7 @@ public class ProvExplicitLink { private boolean shared = false; private boolean accessible = false; private boolean deleted = false; - private Function idProvider; + private Function idProvider; public ProvExplicitLink() { } @@ -39,7 +39,7 @@ public T getNode() { return node; } - public void setNode(T node, Function idProvider) { + public void setNode(T node, Function idProvider) { this.node = node; this.idProvider = idProvider; } @@ -112,7 +112,7 @@ public void addDownstream(ProvExplicitLink node) { downstream.add(node); } - public String getNodeId() { + public Integer getNodeId() { return idProvider.apply(node); } } diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/models/version/ModelVersion.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/models/version/ModelVersion.java index 9386523b8c..8af2ed4dda 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/models/version/ModelVersion.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/models/version/ModelVersion.java @@ -118,6 +118,14 @@ public class ModelVersion implements Serializable { public ModelVersion() { } + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + public Integer getVersion() { return version; } diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/provenance/ModelLink.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/provenance/ModelLink.java new file mode 100644 index 0000000000..4a60d035d8 --- /dev/null +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/provenance/ModelLink.java @@ -0,0 +1,182 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.persistence.entity.provenance; + +import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; + +import javax.persistence.Basic; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.NamedQueries; +import javax.persistence.NamedQuery; +import javax.persistence.OneToOne; +import javax.persistence.Table; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Objects; + +/** + * Entity class representing the model_link table in Hopsworks database. + * An instance of this class represents a row in the table. + */ +@Entity +@Table(name = "model_link", catalog = "hopsworks") +@XmlRootElement +@NamedQueries({ + @NamedQuery(name = "ModelLink.findByChildren", + query = "SELECT l FROM ModelLink l LEFT JOIN FETCH l.parentTrainingDataset " + + "WHERE l.model IN :children " + + "ORDER BY l.parentTrainingDataset ASC, l.id DESC"), + @NamedQuery(name = "ModelLink.findByParents", + query = "SELECT l FROM ModelLink l LEFT JOIN FETCH l.model " + + "WHERE l.parentTrainingDataset IN :parents " + + "ORDER BY l.model.model.name ASC, l.model.version DESC, l.id DESC")}) +public class ModelLink implements ProvExplicitNode, Serializable { + + private static final long serialVersionUID = 1L; + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Basic(optional = false) + @Column(name = "id") + private Integer id; + @JoinColumn(name = "model_version_id", referencedColumnName = "id") + @OneToOne(fetch = FetchType.LAZY) + private ModelVersion model; + @JoinColumn(name = "parent_training_dataset_id", referencedColumnName = "id") + @OneToOne(fetch = FetchType.LAZY) + private TrainingDataset parentTrainingDataset; + @Basic(optional = false) + @Column(name = "parent_feature_store") + private String parentFeatureStore; + @Basic(optional = false) + @Column(name = "parent_feature_view_name") + private String parentFeatureViewName; + @Basic(optional = false) + @Column(name = "parent_feature_view_version") + private Integer parentFeatureViewVersion; + @Basic(optional = false) + @Column(name = "parent_training_dataset_version") + private Integer parentTrainingDatasetVersion; + + public ModelLink() {} + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public ModelVersion getModel() { + return model; + } + + public void setModel(ModelVersion model) { + this.model = model; + } + + public TrainingDataset getParentTrainingDataset() { + return parentTrainingDataset; + } + + public void setParentTrainingDataset(TrainingDataset parentTrainingDataset) { + this.parentTrainingDataset = parentTrainingDataset; + } + + public String getParentFeatureStore() { + return parentFeatureStore; + } + + public void setParentFeatureStore(String parentFeatureStore) { + this.parentFeatureStore = parentFeatureStore; + } + + public String getParentFeatureViewName() { + return parentFeatureViewName; + } + + public void setParentFeatureViewName(String parentFeatureViewName) { + this.parentFeatureViewName = parentFeatureViewName; + } + + public Integer getParentFeatureViewVersion() { + return parentFeatureViewVersion; + } + + public void setParentFeatureViewVersion(Integer parentFeatureViewVersion) { + this.parentFeatureViewVersion = parentFeatureViewVersion; + } + + public Integer getParentTrainingDatasetVersion() { + return parentTrainingDatasetVersion; + } + + public void setParentTrainingDatasetVersion(Integer parentTrainingDatasetVersion) { + this.parentTrainingDatasetVersion = parentTrainingDatasetVersion; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ModelLink)) { + return false; + } + ModelLink modelLink = (ModelLink) o; + return id.equals(modelLink.id) && + model.equals(modelLink.model) && + parentTrainingDataset.equals(modelLink.parentTrainingDataset) && + parentFeatureStore.equals(modelLink.parentFeatureStore) && + parentFeatureViewName.equals(modelLink.parentFeatureViewName) && + parentFeatureViewVersion.equals(modelLink.parentFeatureViewVersion) && + parentTrainingDatasetVersion.equals(modelLink.parentTrainingDatasetVersion); + } + + @Override + public int hashCode() { + return Objects.hash(id, model, parentTrainingDataset, parentFeatureStore, parentFeatureViewName, + parentFeatureViewVersion, parentTrainingDatasetVersion); + } + + @Override + public String parentProject() { + return parentFeatureStore; + } + + @Override + public String parentName() { + return parentFeatureViewName + "_" + parentFeatureViewVersion; + } + + @Override + public Integer parentVersion() { + return parentTrainingDatasetVersion; + } + + @Override + public Integer parentId() { + return parentTrainingDataset != null ? parentTrainingDataset.getId() : null; + } +} \ No newline at end of file diff --git a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/provenance/ProvExplicitNode.java b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/provenance/ProvExplicitNode.java index 7621068495..0fa3bfa6ce 100644 --- a/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/provenance/ProvExplicitNode.java +++ b/hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/provenance/ProvExplicitNode.java @@ -18,9 +18,9 @@ public interface ProvExplicitNode { enum Type { FEATURE_GROUP, - EXTERNAL_FEATURE_GROUP, FEATURE_VIEW, - TRAINING_DATASET; + TRAINING_DATASET, + MODEL; } String parentProject(); diff --git a/hopsworks-persistence/src/main/resources/META-INF/persistence.xml b/hopsworks-persistence/src/main/resources/META-INF/persistence.xml index 3a670c3236..2d953c4043 100644 --- a/hopsworks-persistence/src/main/resources/META-INF/persistence.xml +++ b/hopsworks-persistence/src/main/resources/META-INF/persistence.xml @@ -138,6 +138,7 @@ io.hops.hopsworks.persistence.entity.project.jobs.DefaultJobConfiguration io.hops.hopsworks.persistence.entity.provenance.FeatureGroupLink io.hops.hopsworks.persistence.entity.provenance.FeatureViewLink + io.hops.hopsworks.persistence.entity.provenance.ModelLink io.hops.hopsworks.persistence.entity.python.PythonEnvironment io.hops.hopsworks.persistence.entity.python.PythonDep io.hops.hopsworks.persistence.entity.python.CondaCommands