Mono/dev

.github/workflows/increment_version_dev.yaml

@@ -24,11 +24,11 @@ jobs:
       - name: Increment versions in pyproject.toml
         run: |
           set -x
-          
+
           echo "Incrementing versions..."
           find . -name "pyproject.toml" | while read -r pyproject; do
               echo "Processing $pyproject"
-          
+
               # Extract current version
               CURRENT_VERSION=$(python -c "
           import tomlkit
@@ -43,31 +43,30 @@ jobs:
               print(f'Error reading version from {pyproject}: {e}', end='')
               exit(1)
           ")
-          
+
               echo "Extracted CURRENT_VERSION: $CURRENT_VERSION"
-          
+
               if [ -z "$CURRENT_VERSION" ]; then
                   echo "Error: Could not extract the current version from $pyproject"
                   cat "$pyproject"
                   continue
               fi
-          
+
               # Increment version
               BASE_VERSION=$(echo "$CURRENT_VERSION" | sed -E 's/(.*)-dev.*/\1/')
               DEV_PART=$(echo "$CURRENT_VERSION" | grep -oE 'dev[0-9]+$' | grep -oE '[0-9]+')
-
+
+              # Fallback if no DEV_PART is found
               if [ -z "$DEV_PART" ]; then
                   DEV_PART=0
               fi
-          
+
               NEW_DEV_PART=$((DEV_PART + 1))
               NEW_VERSION="${BASE_VERSION}-dev${NEW_DEV_PART}"
-          
+
               echo "Updating version from $CURRENT_VERSION to $NEW_VERSION"
           done
 
-
-
       - name: Commit changes
         run: |
           git config user.name "github-actions[bot]"

.github/workflows/sequence_publish.yaml

@@ -93,7 +93,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Wait for swarmauri
-        run: sleep 60
+        run: sleep 120
 
       - name: Set up Python 3.12
         uses: actions/setup-python@v5

.github/workflows/test_changed_files.yaml

@@ -126,7 +126,7 @@ jobs:
       - name: Install package dependencies
         run: |
           cd pkgs/${{ matrix.package_tests.package }}
-          poetry install --no-cache -vv
+          poetry install --no-cache --all-extras -vv
 
       - name: Run all tests for the package
         run: |

pkgs/community/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "swarmauri-community"
-version = "0.5.2.dev20"
+version = "0.5.3.dev5"
 description = "This repository includes Swarmauri community components."
 authors = ["Jacob Stewart <[email protected]>"]
 license = "Apache-2.0"
@@ -15,48 +15,65 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"
-captcha = "*"
-chromadb = "*"
-duckdb = "*"
-folium = "*"
-gensim = "*"
-#google-generativeai = "*"
-gradio = "*"
-leptonai = "0.22.0"
-neo4j = "*"
-nltk = "*"
-#openai = "^1.52.0"
-pandas = "*"
-psutil = "*"
-pygithub = "*"
-python-dotenv = "*"
-qrcode = "*"
-redis = "^4.0"
-scikit-learn="^1.4.2"
-swarmauri = "==0.5.2"
-textstat = "*"
-transformers = ">=4.45.0"
-typing_extensions = "*"
-tiktoken = "*"
-pymupdf = "*"
-annoy = "*"
-qdrant_client = "*"
-weaviate = "*"
-pinecone-client = { version = "*", extras = ["grpc"] }
-PyPDF2 = "*"
-pypdftk = "*"
-weaviate-client = "*"
-protobuf = "^3.20.0"
-# Pacmap requires specific version of numba
-#numba = ">=0.59.0"
-#pacmap = "==0.7.3"
+captcha = "^0.6.0"
+chromadb = { version = "^0.5.17", optional = true }
+duckdb = { version = "^1.1.1", optional = true }
+folium = { version = "^0.18.0", optional = true }
+gensim = { version = "^4.3.3", optional = true }
+gradio = { version = "^5.4.0", optional = true }
+leptonai = { version = "^0.22.0", optional = true }
+neo4j = { version = "^5.25.0", optional = true }
+nltk = { version = "^3.9.1", optional = true }
+pandas = "^2.2.3"
+psutil = { version = "^6.1.0", optional = true }
+pygithub = { version = "^2.4.0", optional = true }
+qrcode = { version = "^8.0", optional = true }
+redis = { version = "^4.0", optional = true }
+swarmauri = "==0.5.3.dev5"
+textstat = { version = "^0.7.4", optional = true }
+transformers = { version = ">=4.45.0", optional = true }
+typing_extensions = "^4.12.2"
+tiktoken = { version = "^0.8.0", optional = true }
+PyMuPDF = { version = "^1.24.12", optional = true }
+annoy = { version = "^1.17.3", optional = true }
+qdrant-client = { version = "^1.12.0", optional = true }
+pinecone-client = { version = "^5.0.1", optional = true, extras = ["grpc"] }
+pypdf = { version = "^5.0.1", optional = true }
+pypdftk = { version = "^0.5", optional = true }
+weaviate-client = { version = "^4.9.2", optional = true }
+#protobuf = { version = "^3.20.0", optional = true }
+
+[tool.poetry.extras]
+# Grouped optional dependencies
+nlp = ["nltk", "gensim", "textstat"]
+ml_toolkits = ["transformers", "annoy"]
+visualization = ["folium"]
+storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate", "pinecone-client"]
+document_processing = ["pypdf", "PyMuPDF", "pypdftk"]
+cloud_integration = ["psutil", "qrcode", "pygithub"]
+gradio = ["gradio"]
+model_clients = ["leptonai"]
+tiktoken = ["tiktoken"]
 
+# Full installation
+full = [
+    "nltk", "gensim", "textstat",
+    "transformers", "annoy",
+    "folium",
+    "redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate", "pinecone-client",
+    "pypdf", "PyMuPDF", "pypdftk",
+    "psutil", "qrcode", "pygithub",
+    "gradio",
+    "leptonai",
+    "tiktoken"
+]
 
 [tool.poetry.dev-dependencies]
-flake8 = "^7.0"  # Add flake8 as a development dependency
-pytest = "^8.0"  # Ensure pytest is also added if you run tests
+flake8 = "^7.0"
+pytest = "^8.0"
 pytest-asyncio = ">=0.24.0"
 pytest-xdist = "^3.6.1"
+python-dotenv = "*"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -70,12 +87,10 @@ markers = [
     "unit: Unit tests",
     "integration: Integration tests",
     "acceptance: Acceptance tests",
-    "experimental: Experimental tests",
+    "experimental: Experimental tests"
 ]
-
 log_cli = true
 log_cli_level = "INFO"
 log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
 log_cli_date_format = "%Y-%m-%d %H:%M:%S"
-
 asyncio_default_fixture_loop_scope = "function"

pkgs/community/swarmauri_community/document_stores/concrete/__init__.py

@@ -1,3 +1,10 @@
-from swarmauri_community.document_stores.concrete.RedisDocumentStore import (
-    RedisDocumentStore,
-)
+from swarmauri.utils._lazy_import import _lazy_import
+
+documents_stores_files = [
+    ("swarmauri_community.documents_stores.concrete.RedisDocumentStore", "RedisDocumentStore"),
+]
+
+for module_name, class_name in documents_stores_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in documents_stores_files]

pkgs/community/swarmauri_community/embeddings/concrete/__init__.py

@@ -0,0 +1,12 @@
+from swarmauri.utils._lazy_import import _lazy_import
+
+
+embeddings_files = [
+    ("swarmauri_community.embeddings.concrete.Doc2VecEmbedding", "Doc2VecEmbedding"),
+    ("swarmauri_community.embeddings.concrete.MlmEmbedding", "MlmEmbedding"),
+]
+
+for module_name, class_name in embeddings_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in embeddings_files]

pkgs/community/swarmauri_community/llms/concrete/__init__.py

@@ -1,4 +1,12 @@
-from swarmauri_community.llms.concrete.LeptonAIImgGenModel import LeptonAIImgGenModel
-from swarmauri_community.llms.concrete.LeptonAIModel import LeptonAIModel
+from swarmauri.utils._lazy_import import _lazy_import
 
-__all__ = ["LeptonAIImgGenModel", "LeptonAIModel"]
+llms_files = [
+    ("swarmauri_community.llms.concrete.LeptonAIImgGenModel", "LeptonAIImgGenModel"),
+    ("swarmauri_community.llms.concrete.LeptonAIModel", "LeptonAIModel"),
+    ("swarmauri_community.llms.concrete.PytesseractImg2TextModel", "PytesseractImg2TextModel"),
+]
+
+for module_name, class_name in llms_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in llms_files]

pkgs/community/swarmauri_community/measurements/concrete/__init__.py

@@ -1,6 +1,11 @@
-from swarmauri_community.measurements.concrete.MutualInformationMeasurement import (
-    MutualInformationMeasurement,
-)
-from swarmauri_community.measurements.concrete.TokenCountEstimatorMeasurement import (
-    TokenCountEstimatorMeasurement,
-)
+from swarmauri.utils._lazy_import import _lazy_import
+
+measurement_files = [
+    ("swarmauri_community.measurements.concrete.MutualInformationMeasurement", "MutualInformationMeasurement"),
+    ("swarmauri_community.measurements.concrete.TokenCountEstimatorMeasurement", "TokenCountEstimatorMeasurement"),
+]
+
+for module_name, class_name in measurement_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in measurement_files]

pkgs/community/swarmauri_community/parsers/concrete/__init__.py

@@ -1,3 +1,16 @@
-from swarmauri_community.parsers.concrete.FitzPdfParser import PDFtoTextParser
-from swarmauri_community.parsers.concrete.PyPDF2Parser import PyPDF2Parser
-from swarmauri_community.parsers.concrete.PyPDFTKParser import PyPDFTKParser
+from swarmauri.utils._lazy_import import _lazy_import
+
+parsers_files = [
+    ("swarmauri_community.parsers.concrete.BERTEmbeddingParser", "BERTEmbeddingParser"),
+    ("swarmauri_community.parsers.concrete.EntityRecognitionParser", "EntityRecognitionParser"),
+    ("swarmauri_community.parsers.concrete.FitzPdfParser", "FitzPdfParser"),
+    ("swarmauri_community.parsers.concrete.PyPDF2Parser", "PyPDF2Parser"),
+    ("swarmauri_community.parsers.concrete.PyPDFTKParser", "PyPDFTKParser"),
+    ("swarmauri_community.parsers.concrete.TextBlobNounParser", "TextBlobNounParser"),
+    ("swarmauri_community.parsers.concrete.TextBlobSentenceParser", "TextBlobSentenceParser"),
+]
+
+for module_name, class_name in parsers_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in parsers_files]

pkgs/community/swarmauri_community/retrievers/concrete/__init__.py

@@ -1,5 +1,10 @@
-# -*- coding: utf-8 -*-
+from swarmauri.utils._lazy_import import _lazy_import
 
-from swarmauri_community.retrievers.concrete.RedisDocumentRetriever import (
-    RedisDocumentRetriever,
-)
+retriever_files = [
+    ("swarmauri_community.retrievers.concrete.RedisDocumentRetriever", "RedisDocumentRetriever"),
+]
+
+for module_name, class_name in retriever_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in retriever_files]

pkgs/community/swarmauri_community/toolkits/concrete/__init__.py

@@ -1 +1,10 @@
-from swarmauri_community.toolkits.concrete.GithubToolkit import *
+from swarmauri.utils._lazy_import import _lazy_import
+
+toolkits_files = [
+    ("swarmauri_community.toolkits.concrete.GithubToolkit", "GithubToolkit"),
+]
+
+for module_name, class_name in toolkits_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in toolkits_files]

pkgs/community/swarmauri_community/tools/concrete/__init__.py

@@ -1,31 +1,33 @@
-from swarmauri_community.tools.concrete.CaptchaGeneratorTool import CaptchaGeneratorTool
-from swarmauri_community.tools.concrete.DaleChallReadabilityTool import (
-    DaleChallReadabilityTool,
-)
-from swarmauri_community.tools.concrete.DownloadPdfTool import DownloadPDFTool
-from swarmauri_community.tools.concrete.EntityRecognitionTool import (
-    EntityRecognitionTool,
-)
-from swarmauri_community.tools.concrete.FoliumTool import FoliumTool
-from swarmauri_community.tools.concrete.GithubBranchTool import GithubBranchTool
-from swarmauri_community.tools.concrete.GithubCommitTool import GithubCommitTool
-from swarmauri_community.tools.concrete.GithubIssueTool import GithubIssueTool
-from swarmauri_community.tools.concrete.GithubPRTool import GithubPRTool
-from swarmauri_community.tools.concrete.GithubRepoTool import GithubRepoTool
-from swarmauri_community.tools.concrete.GithubTool import GithubTool
-from swarmauri_community.tools.concrete.GmailReadTool import GmailReadTool
-from swarmauri_community.tools.concrete.GmailSendTool import GmailSendTool
-from swarmauri_community.tools.concrete.LexicalDensityTool import LexicalDensityTool
-from swarmauri_community.tools.concrete.PsutilTool import PsutilTool
-from swarmauri_community.tools.concrete.QrCodeGeneratorTool import QrCodeGeneratorTool
-from swarmauri_community.tools.concrete.SentenceComplexityTool import (
-    SentenceComplexityTool,
-)
-from swarmauri_community.tools.concrete.SentimentAnalysisTool import (
-    SentimentAnalysisTool,
-)
-from swarmauri_community.tools.concrete.SMOGIndexTool import SMOGIndexTool
-from swarmauri_community.tools.concrete.WebScrapingTool import WebScrapingTool
-from swarmauri_community.tools.concrete.ZapierHookTool import ZapierHookTool
+from swarmauri.utils._lazy_import import _lazy_import
 
-# from swarmauri_community.tools.concrete.PaCMAPTool import PaCMAPTool
+tool_files = [
+    ("swarmauri_community.tools.concrete.CaptchaGeneratorTool", "CaptchaGeneratorTool"),
+    ("swarmauri_community.tools.concrete.DaleChallReadabilityTool", "DaleChallReadabilityTool"),
+    ("swarmauri_community.tools.concrete.DownloadPdfTool", "DownloadPDFTool"),
+    ("swarmauri_community.tools.concrete.EntityRecognitionTool", "EntityRecognitionTool"),
+    ("swarmauri_community.tools.concrete.FoliumTool", "FoliumTool"),
+    ("swarmauri_community.tools.concrete.GithubBranchTool", "GithubBranchTool"),
+    ("swarmauri_community.tools.concrete.GithubCommitTool", "GithubCommitTool"),
+    ("swarmauri_community.tools.concrete.GithubIssueTool", "GithubIssueTool"),
+    ("swarmauri_community.tools.concrete.GithubPRTool", "GithubPRTool"),
+    ("swarmauri_community.tools.concrete.GithubRepoTool", "GithubRepoTool"),
+    ("swarmauri_community.tools.concrete.GithubTool", "GithubTool"),
+    ("swarmauri_community.tools.concrete.GmailReadTool", "GmailReadTool"),
+    ("swarmauri_community.tools.concrete.GmailSendTool", "GmailSendTool"),
+    ("swarmauri_community.tools.concrete.LexicalDensityTool", "LexicalDensityTool"),
+    ("swarmauri_community.tools.concrete.PsutilTool", "PsutilTool"),
+    ("swarmauri_community.tools.concrete.QrCodeGeneratorTool", "QrCodeGeneratorTool"),
+    ("swarmauri_community.tools.concrete.SentenceComplexityTool", "SentenceComplexityTool"),
+    ("swarmauri_community.tools.concrete.SentimentAnalysisTool", "SentimentAnalysisTool"),
+    ("swarmauri_community.tools.concrete.SMOGIndexTool", "SMOGIndexTool"),
+    ("swarmauri_community.tools.concrete.WebScrapingTool", "WebScrapingTool"),
+    ("swarmauri_community.tools.concrete.ZapierHookTool", "ZapierHookTool"),
+    # ("swarmauri_community.tools.concrete.PaCMAPTool", "PaCMAPTool"),
+]
+
+# Lazy loading of tools, storing them in variables
+for module_name, class_name in tool_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+# Adding the lazy-loaded tools to __all__
+__all__ = [class_name for _, class_name in tool_files]