Merge pull request #195 from permitio/release/v0.7.0

Release/v0.7.0

.github/workflows/pdp_cicd.yml

@@ -33,12 +33,22 @@ jobs:
         cd ./permit-opa
         find * -name '*go*' -print0 | xargs -0 tar -czf "$build_root"/custom/custom_opa.tar.gz --exclude '.*'
 
+    - uses: robinraju/release-downloader@v1
+      with:
+        repository: permitio/datasync
+        latest: true
+        fileName: factstore_server*
+        token: ${{ secrets.CLONE_REPO_TOKEN }}
+        out-file-path: "factdb"
+
     - name: Build and load image for PDP E2E tests
       uses: docker/build-push-action@v5
       with:
         push: false
         load: true
         context: .
+        build-args: |
+          ALLOW_MISSING_FACTSTORE=false
         platforms: linux/amd64
         tags: permitio/pdp-v2:test
         cache-from: type=gha
@@ -147,13 +157,23 @@ jobs:
         cd ./permit-opa
         find * -name '*go*' -print0 | xargs -0 tar -czf "$build_root"/custom/custom_opa.tar.gz --exclude '.*'
 
+    - uses: robinraju/release-downloader@v1
+      with:
+        repository: permitio/datasync
+        latest: true
+        fileName: factstore_server*
+        token: ${{ secrets.CLONE_REPO_TOKEN }}
+        out-file-path: "factdb"
+
     - name: Build and push PDP image - (pre-release)
       if: "github.event.release.prerelease"
       uses: docker/build-push-action@v5
       with:
         push: true
         context: .
         platforms: linux/amd64,linux/arm64
+        build-args: |
+          ALLOW_MISSING_FACTSTORE=false
         tags: permitio/pdp-v2:${{ github.event.release.tag_name }}
         cache-from: type=gha
         cache-to: type=gha,mode=max
@@ -165,6 +185,8 @@ jobs:
         push: true
         context: .
         platforms: linux/amd64,linux/arm64
+        build-args: |
+          ALLOW_MISSING_FACTSTORE=false
         tags: permitio/pdp-v2:${{ github.event.release.tag_name }},permitio/pdp-v2:latest
         cache-from: type=gha
         cache-to: type=gha,mode=max

Dockerfile

@@ -4,27 +4,70 @@
 FROM golang:bullseye AS opa_build
 
 COPY custom* /custom
+COPY factdb* /factdb
 
 RUN if [ -f /custom/custom_opa.tar.gz ]; \
-    then \
-      cd /custom && \
-      tar xzf custom_opa.tar.gz && \
-      go build -o /opa && \
-      rm -rf /custom ; \
+  then \
+  cd /custom && \
+  tar xzf custom_opa.tar.gz && \
+  go build -ldflags="-extldflags=-static" -o /opa && \
+  rm -rf /custom ; \
+  else \
+  case $(uname -m) in \
+  x86_64) \
+  curl -L -o /opa https://openpolicyagent.org/downloads/latest/opa_linux_amd64_static ; \
+  ;; \
+  aarch64) \
+  curl -L -o /opa https://openpolicyagent.org/downloads/latest/opa_linux_arm64_static ; \
+  ;; \
+  *) \
+  echo "Unknown architecture." ; \
+  exit 1 ; \
+  ;; \
+  esac ; \
+  fi
+
+RUN if [ -f /factdb/factdb.tar.gz ]; \
+  then \
+  cd /factdb && \
+  tar xzf factdb.tar.gz && \
+  go build -ldflags="-extldflags=-static" -o /bin/factdb ./cmd/factstore_server && \
+  rm -rf /factdb ; \
+  else \
+  case $(uname -m) in \
+  x86_64) \
+    if [ -f /factdb/factstore_server-linux-amd64 ]; then \
+      cp /factdb/factstore_server-linux-amd64 /bin/factdb; \
     else \
-      case $(uname -m) in \
-      	x86_64) \
-      		curl -L -o /opa https://openpolicyagent.org/downloads/latest/opa_linux_amd64_static ; \
-      		;; \
-      	aarch64) \
-      		curl -L -o /opa https://openpolicyagent.org/downloads/latest/opa_linux_arm64_static ; \
-      		;; \
-      	*) \
-      		echo "Unknown architecture." ; \
-      		exit 1 ; \
-      		;; \
-      esac ; \
-    fi
+      echo "factstore_server-linux-amd64 not found." ; \
+      if [ "$ALLOW_MISSING_FACTSTORE" = "false" ]; then \
+        echo "Missing Factstore is not allowed, exiting..."; exit 1; \
+      else \
+        echo "Missing Factstore is allowed, continuing..."; \
+        touch /bin/factdb ; \
+      fi \
+    fi \
+  ;; \
+  aarch64) \
+    if [ -f /factdb/factstore_server-linux-arm64 ]; then \
+      cp /factdb/factstore_server-linux-arm64 /bin/factdb; \
+    else \
+      echo "factstore_server-linux-arm64 not found." ; \
+      if [ "$ALLOW_MISSING_FACTSTORE" = "false" ]; then \
+        echo "Missing Factstore is not allowed, exiting..."; exit 1; \
+      else \
+        echo "Missing Factstore is allowed, continuing..."; \
+        touch /bin/factdb ; \
+      fi \
+    fi \
+  ;; \
+  *) \
+    echo "Unknown architecture." ; \
+    exit 1 ; \
+  ;; \
+  esac ; \
+  fi
+
 
 # MAIN IMAGE ----------------------------------------
 # most of the time only this image should be built
@@ -49,6 +92,9 @@ RUN chown -R permit:permit /app/bin
 COPY --from=opa_build --chmod=755 /opa /app/bin/opa
 ENV OPAL_INLINE_OPA_EXEC_PATH="/app/bin/opa"
 
+COPY --from=opa_build --chmod=755 /bin/factdb /app/bin/factdb
+ENV PDP_FACTDB_BINARY_PATH="/app/bin/factdb"
+
 # bash is needed for ./start/sh script
 COPY scripts ./
 
@@ -115,12 +161,14 @@ ENV PDP_API_KEY="MUST BE DEFINED"
 ENV PDP_REMOTE_CONFIG_ENDPOINT="/v2/pdps/me/config"
 ENV PDP_REMOTE_STATE_ENDPOINT="/v2/pdps/me/state"
 ENV PDP_VERSION_FILE_PATH="/app/permit_pdp_version"
+ENV PDP_FACTDB_BINARY_PATH="/app/bin/factdb"
 # This is a default PUBLIC (not secret) key,
 # and it is here as a safety measure on purpose.
 ENV OPAL_AUTH_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDe2iQ+/E01P2W5/EZwD5NpRiSQ8/r/k18pFnym+vWCSNMWpd9UVpgOUWfA9CAX4oEo5G6RfVVId/epPH/qVSL87uh5PakkLZ3E+PWVnYtbzuFPs/lHZ9HhSqNtOQ3WcPDTcY/ST2jyib2z0sURYDMInSc1jnYKqPQ6YuREdoaNdPHwaTFN1tEKhQ1GyyhL5EDK97qU1ejvcYjpGm+EeE2sjauHYn2iVXa2UA9fC+FAKUwKqNcwRTf3VBLQTE6EHGWbxVzXv1Feo8lPZgL7Yu/UPgp7ivCZhZCROGDdagAfK9sveYjkKiWCLNUSpado/E5Vb+/1EVdAYj6fCzk45AdQzA9vwZefP0sVg7EuZ8VQvlz7cU9m+XYIeWqduN4Qodu87rtBYtSEAsru/8YDCXBDWlLJfuZb0p/klbte3TayKnQNSWD+tNYSJHrtA/3ZewP+tGDmtgLeB38NLy1xEsgd31v6ISOSCTHNS8ku9yWQXttv0/xRnuITr8a3TCLuqtUrNOhCx+nKLmYF2cyjYeQjOWWpn/Z6VkZvOa35jhG1ETI8IwE+t5zXqrf2s505mh18LwA1DhC8L/wHk8ZG7bnUe56QwxEo32myUBN8nHdu7XmPCVP8MWQNLh406QRAysishWhXVs/+0PbgfBJ/FxKP8BXW9zqzeIG+7b/yk8tRHQ=="
 # expose sidecar port
 EXPOSE 7000
 # expose opa directly
 EXPOSE 8181
+
 # run gunicorn
 CMD ["/app/start.sh"]

Makefile

@@ -21,7 +21,7 @@ build-amd64: prepare
 	@docker buildx build --platform linux/amd64 -t permitio/pdp-v2:$(VERSION) . --load
 
 build-arm64: prepare
-	@docker buildx build --platform linux/arm64 -t permitio/pdp-v2:$(VERSION) . --load
+	@docker buildx build --build-arg ALLOW_MISSING_FACTSTORE=false --platform linux/arm64 -t permitio/pdp-v2:$(VERSION) . --load
 
 run: run-prepare
 	@docker run -it --rm -p 7766:7000 --env PDP_API_KEY=$(API_KEY) --env PDP_DEBUG=true permitio/pdp-v2:$(VERSION)

build_opal_bundle.sh

@@ -15,6 +15,14 @@ else
   echo "permit-opa directory already exists. Skipping clone operation."
 fi
 
+# Check if factdb directory already exists
+if [ ! -d "../factdb" ]; then
+  # Clone the permit-opa repository into the parent directory if it doesn't exist
+  git clone [email protected]:permitio/datasync.git ../factdb
+else
+  echo "factdb directory already exists. Skipping clone operation."
+fi
+
 # Conditionally execute the custom OPA tarball creation section based on the value of PDP_VANILLA
 if [ "$PDP_VANILLA" != "true" ]; then
   # Custom OPA tarball creation section
@@ -24,6 +32,13 @@ if [ "$PDP_VANILLA" != "true" ]; then
   cd "../permit-opa"
   find * -name '*go*' -print0 | xargs -0 tar -czf "$build_root"/custom/custom_opa.tar.gz --exclude '.*'
   cd "$build_root"
+  # factdb tarball creation section
+  rm -rf factdb
+  mkdir factdb
+  build_root="$PWD"
+  cd "../factdb"
+  find * -name '*go*' -print0 | xargs -0 tar -czf "$build_root"/factdb/factdb.tar.gz --exclude '.*'
+  cd "$build_root"
 else
   echo "Skipping custom OPA tarball creation for pdp-vanilla environment."
 fi

horizon/config.py

@@ -18,6 +18,33 @@ def __new__(cls, prefix=None, is_model=True):
             cls.instance = super(SidecarConfig, cls).__new__(cls)
         return cls.instance
 
+    FACTDB_ENABLED = confi.bool(
+        "FACTDB_ENABLED",
+        False,
+        description="if true, the sidecar will enable the FactDB service to manage the PDP data in "
+        "FactDB",
+    )
+
+    FACTDB_BINARY_PATH = confi.str(
+        "FACTDB_BINARY_PATH",
+        "/factstore",
+        description="path in which to find the FactDB executable",
+    )
+
+    FACTDB_SERVICE_URL = confi.str(
+        "FACTDB_SERVICE_URL",
+        "http://localhost:8080",
+        description="URL to the FactDB service that manages the PDP data, will only be used if "
+        "FACTDB_ENABLED is true",
+    )
+
+    FACTDB_BACKUP_SERVER_URL = confi.str(
+        "FACTDB_BACKUP_SERVER_URL",
+        None,
+        description="URL from which the FactDB service will fetch the PDP data backup from, will only be used if "
+        "FACTDB_ENABLE_REMOTE_BACKUP is true",
+    )
+
     SHARD_ID = confi.str(
         "SHARD_ID",
         None,