chore(docker): improve local docker and general development setup

closes #229

.env.sandbox.docker-compose-dev

@@ -0,0 +1,20 @@
+# Common
+AkashSandboxDatabaseCS=postgres://postgres:password@db:5432/cloudmos-akash-sandbox
+UserDatabaseCS=postgres://postgres:password@db:5432/cloudmos-users
+Network=sandbox
+ActiveChain=akashSandbox
+
+# Deploy Web
+API_BASE_URL: http://api:3080
+PROVIDER_PROXY_URL: http://provider-proxy:3040
+
+# Stats Web
+API_MAINNET_BASE_URL: http://api:3080
+API_TESTNET_BASE_URL: http://api:3080
+API_SANDBOX_BASE_URL: http://api:3080
+
+# DB
+POSTGRES_USER: postgres
+POSTGRES_PASSWORD: password
+POSTGRES_DBS_FOR_IMPORT: cloudmos-akash-sandbox
+POSTGRES_USERS_DB: cloudmos-users

.github/workflows/docker-build-deploy-web.yml

@@ -24,4 +24,4 @@ jobs:
 
       - name: Build the Docker image
         if: steps.filter.outputs.deploy-web == 'true'
-        run: docker build -f docker/Dockerfile.nextjs --build-arg WORKSPACE=apps/deploy-web -t console-deploy-web .
+        run: npm run dc:build -- deploy-web

.github/workflows/docker-build-indexer.yml

@@ -25,4 +25,4 @@ jobs:
 
       - name: Build the Docker image
         if: steps.filter.outputs.indexer == 'true'
-        run: docker build -f docker/Dockerfile.indexer -t console-indexer .
+        run: npm run dc:build -- indexer

.github/workflows/docker-build-landing.yml

@@ -24,4 +24,4 @@ jobs:
 
       - name: Build the Docker image for LANDING
         if: steps.filter.outputs.landing == 'true'
-        run: docker build -f docker/Dockerfile.landing -t console-landing .
+        run: npm run dc:build -- landing

.github/workflows/docker-build-provider-proxy.yml

@@ -24,4 +24,4 @@ jobs:
 
       - name: Build the Docker image
         if: steps.filter.outputs.provider-proxy == 'true'
-        run: docker build -f docker/Dockerfile.provider-proxy -t console-provider-proxy .
+        run: npm run dc:build -- provider-proxy

.github/workflows/docker-build-stats-web.yml

@@ -24,4 +24,4 @@ jobs:
 
       - name: Build the Docker image
         if: steps.filter.outputs.stats-web == 'true'
-        run: docker build --target production -f docker/Dockerfile.nextjs --build-arg WORKSPACE=apps/stats-web -t console-stats-web .
+        run: npm run dc:build -- stats-web

.github/workflows/validate-n-build-api.yml

@@ -55,4 +55,4 @@ jobs:
 
       - name: Build the Docker image for API
         if: steps.filter.outputs.api == 'true'
-        run: docker build -f docker/Dockerfile.api -t console-api .
+        run: npm run dc:build -- api

.gitignore

@@ -16,52 +16,42 @@ pids
 *.seed
 *.pid.lock
 
-# Directory for instrumented libs generated by jscoverage/JSCover
+# Coverage directories and reports
 lib-cov
-
-# Coverage directory used by tools like istanbul
 coverage
 *.lcov
-
-# nyc test coverage
 .nyc_output
 
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
 # Dependency directories
 node_modules/
 jspm_packages/
-
-# Snowpack dependency directory (https://snowpack.dev/)
 web_modules/
+bower_components
 
-# TypeScript cache
-*.tsbuildinfo
+# Build directories and tools
+build/Release
+.grunt
+.lock-wscript
+.next
+out
+dist
+build.ps1
 
-# Optional npm cache directory
+# Caches
 .npm
-
-# Optional eslint cache
 .eslintcache
-
-# Optional stylelint cache
 .stylelintcache
-
-# Microbundle cache
 .rpt2_cache/
 .rts2_cache_cjs/
 .rts2_cache_es/
 .rts2_cache_umd/
+.turbo
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+*.tsbuildinfo
 
 # Optional REPL history
 .node_repl_history
@@ -79,27 +69,10 @@ web_modules/
 .env.production.local
 .env.local
 
-# Next.js build output
-.next
-out
-dist
-
-# Stores VSCode versions used for testing VSCode extensions
-.vscode-test
-
 # IDE files
 .idea
 .vscode
-
-# yarn v2
-.yarn/cache
-.yarn/unplugged
-.yarn/build-state.yml
-.yarn/install-state.gz
-.pnp.*
-
-# Scripts
-build.ps1
+.vscode-test
 
 # Data Folder
 data

README.md

@@ -15,12 +15,28 @@
 
 </div>
 
+- [Quick Start](#quick-start)
 - [Services](#services)
 - [Monitoring](#monitoring)
 - [Example SQL Queries](#example-sql-queries)
+- [Running the Application](#running-the-application)
 - [How to run](#how-to-run)
 - [Database Structure](#database-structure)
 
+# Quick start
+
+We use `docker` with `docker compose` to run the services. This is the easiest way to get started. For instance to run the deploy-web service:
+
+```bash
+git clone [email protected]:akash-network/cloudmos.git ./console
+cd console
+npm run dc:up:dev -- deploy-web 
+```
+
+This would start the deploy-web service in development mode with all the dependencies (api, indexer, postgres) needed. While spinning up postgres it would also import a backup of the sandbox (by default) database to speed up the process.
+
+For more details on how to run the other services or modes see the [Running the Application](#running-the-application) section.
+
 # Services
 
 ## Dataflow between services
@@ -86,6 +102,43 @@ See [Example_Queries.md](./doc/Example_Queries.md)
 
 This document provides instructions on how to set up and run the application, including steps for manual database restoration and using Docker Compose for ease of setup.
 
+### Using Docker and Docker Compose
+This project's service are deployed using Docker and Docker Compose. The following sections provide instructions for setting up and running the application using Docker Compose.
+All the Dockerfiles are using multi-stage builds to optimize the image build processes. Same files are used to build both development and production images. There are 3 docker-compose files:
+- **docker-compose.build.yml:** Base file solely building production images for the services. It can be used to verify the same build process as in CICD.
+- **docker-compose.prod.yml:** This file is used to run the services in production mode. It also includes the database service which would fetch a remote backup and import it on init.
+- **docker-compose.yml:** The default file to run all the services in development mode with features like hot-reload.
+
+Some commands are added to package.json for convenience.
+
+```shell
+npm run dc:build # Build the production images
+npm run dc:up:prod # Run the services in production mode
+npm run dc:up:dev # Run the services in development mode
+npm run dc:down # Stop the services referencing any possible service
+```
+
+Note: you may pass any `docker compose` related arguments to the above commands. E.g. to only start `deploy-web` service in development mode:
+```shell
+npm run dc:up:dev -- deploy-web
+```
+This would also properly spin up all the dependencies like the `api`.
+### Using Turbo Repo
+Another way to run apps in dev mode is using turbo repo setup. Some available commands are:
+```shell
+npm run console:dev # run console ui in dev mode with dependencies
+npm run stats:dev # run stats ui in dev mode with dependencies
+npm run api:dev # run api in dev mode with dependencies
+npm run indexer:dev # run indexer in dev mode with dependencies
+```
+
+Note the above commands still depend on docker to run postgres database. If you need to run them without db you can use the following commands:
+```shell
+npm run console:dev:no-db # run console ui in dev mode with dependencies but without postgres in docker
+npm run stats:dev:no-db # run stats ui in dev mode with dependencies but without postgres in docker
+```
+
+
 ## Manual Database Restoration
 
 Due to the extensive time required to index Akash from block #1, it's recommended to initialize your database using an existing backup for efficiency. This approach is particularly beneficial for development purposes.
@@ -108,19 +161,6 @@ gunzip -c /path/to/cloudmos-akash-sandbox.sql.gz | psql --host "localhost" --por
 ```
 After restoring the database, you can proceed with the specific project's README instructions for further setup and running the application.
 
-### Using Docker Compose
-For convenience, a Docker Compose configuration is provided to automate the database setup, download, and data import process.
-
-#### Configuration
-1. Ensure environment variables are set in .postgres.local.env. This file configures PostgreSQL and import settings.
-2. The `POSTGRES_DBS_FOR_IMPORT` variable should be updated with the databases you wish to import. This is a comma-separated list with potential values including `cloudmos-akash-sandbox` (default) and `cloudmos-akash-2` (mainnet). Leave this variable empty if no import is desired.
-
-#### Running Docker Compose
-```sh
-docker-compose up db
-```
-This command spins up the database service and automatically handles the downloading and importing of the specified data.
-
 # Database Structure
 
 The database schemas is defined using [sequelize-typescript](https://github.com/sequelize/sequelize-typescript) in [/shared/dbSchemas/](./shared/dbSchemas/). Models are separated into the following folders:

apps/api/package.json

@@ -14,6 +14,7 @@
     "format": "prettier --write ./*.{js,json} **/*.{ts,js,json}",
     "lint": "eslint .",
     "start": "webpack --config webpack.dev.js --watch",
+    "dev": "npm run start",
     "test": "jest --selectProjects unit functional",
     "test:cov": "jest --selectProjects unit functional --coverage",
     "test:functional": "jest --selectProjects functional",
@@ -40,7 +41,6 @@
     "@octokit/rest": "^18.12.0",
     "@sentry/node": "^7.55.2",
     "axios": "^0.27.2",
-    "bufferutil": "^4.0.8",
     "cosmjs-types": "^0.5.0",
     "date-fns": "^2.29.2",
     "date-fns-tz": "^1.3.6",
@@ -59,7 +59,6 @@
     "sequelize": "^6.21.3",
     "sequelize-typescript": "^2.1.5",
     "stripe": "^10.14.0",
-    "utf-8-validate": "^5.0.10",
     "uuid": "^9.0.1",
     "zod": "^3.22.4"
   },

apps/api/webpack.dev.js

@@ -1,10 +1,11 @@
 const path = require("path");
-const { NODE_ENV = "development" } = process.env;
 const NodemonPlugin = require("nodemon-webpack-plugin");
 const nodeExternals = require("webpack-node-externals");
 const hq = require("alias-hq");
 const webpack = require("webpack");
 
+const { NODE_ENV = "development" } = process.env;
+
 module.exports = {
   entry: "./src/index.ts",
   mode: NODE_ENV,
@@ -24,8 +25,7 @@ module.exports = {
       {
         test: /\.(ts|js)x?$/,
         exclude: /node_modules/,
-        loader: "ts-loader",
-        options: { configFile: "tsconfig.json" }
+        loader: "ts-loader"
       }
     ]
   },

apps/api/webpack.prod.js

@@ -1,8 +1,9 @@
 const path = require("path");
-const { NODE_ENV = "production" } = process.env;
+const webpack = require("webpack");
 const nodeExternals = require("webpack-node-externals");
 const hq = require("alias-hq");
-const webpack = require("webpack");
+
+const { NODE_ENV = "production" } = process.env;
 
 module.exports = {
   entry: "./src/index.ts",
@@ -22,11 +23,13 @@ module.exports = {
       {
         test: /\.(ts|js)x?$/,
         exclude: /node_modules/,
-        loader: "ts-loader",
-        options: { configFile: "tsconfig.json" }
+        loader: "ts-loader"
       }
     ]
   },
+  optimization: {
+    minimize: false
+  },
   plugins: [new webpack.IgnorePlugin({ resourceRegExp: /^pg-native$/ })],
   node: {
     __dirname: true

apps/deploy-web/nginx.conf

@@ -7,7 +7,6 @@ http {
     server {
         # Redirect HTTP requests to HTTPS.
         listen 80;
-
         return 307 https://$host$request_uri;
     }
 
@@ -18,14 +17,14 @@ http {
 
         ssl_certificate /etc/nginx/ssl/my_ssl_cert.crt;
         ssl_certificate_key /etc/nginx/ssl/my_ssl_key.key;
-        
+
         location / {
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto https;
             proxy_set_header X-Forwarded-Ssl on;
             proxy_set_header Host $http_host;
             proxy_redirect off;
-            proxy_pass http://127.0.0.1:3001;
+            proxy_pass http://127.0.0.1:3000;
             proxy_buffers 8 16k;
             proxy_buffer_size 16k;
             proxy_cookie_path / "/; HTTPOnly; Secure";

apps/deploy-web/package.json

@@ -8,7 +8,7 @@
   "scripts": {
     "build": "next build",
     "build-analyze": "set ANALYZE=true&& next build",
-    "dev": "next -p 3000",
+    "dev": "next",
     "format": "prettier --write ./*.{ts,js,json} **/*.{ts,tsx,js,json}",
     "lint": "eslint .",
     "start": "next start",
@@ -124,7 +124,6 @@
     "eslint": "^8.57.0",
     "eslint-config-next": "^14.2.3",
     "eslint-plugin-simple-import-sort": "^12.1.0",
-    "patch-package": "^8.0.0",
     "postcss": "^8.4.31",
     "postcss-nesting": "^12.0.2",
     "prettier": "^3.3.0",

apps/indexer/package.json

@@ -18,6 +18,7 @@
     "format": "prettier --write ./*.{js,json} **/*.{ts,js,json}",
     "lint": "eslint .",
     "start": "webpack --mode development --config webpack.dev.js --watch",
+    "dev": "npm run start",
     "test": "jest"
   },
   "dependencies": {

apps/indexer/webpack.dev.js

@@ -1,14 +1,16 @@
 const path = require("path");
-const { NODE_ENV = "development" } = process.env;
 const NodemonPlugin = require("nodemon-webpack-plugin");
 const nodeExternals = require("webpack-node-externals");
 const hq = require("alias-hq");
 const webpack = require("webpack");
 
+const { NODE_ENV = "development" } = process.env;
+
 module.exports = {
   entry: "./src/index.ts",
   mode: NODE_ENV,
   target: "node",
+  devtool: "source-map",
   output: {
     path: path.resolve(__dirname, "dist"),
     filename: "server.js"