Adding commands, eventually for pow

- Including empty Git repo creation

- Making optimizations in various places. Adding docstrings (#350)

- Making it possible to do RealSimpleProperty decls in class body.
  This was added now mostly for documentation purposes, but also because
  eventually, we'll want to add these relationships to the graph (#351)

- Reordering operations to avoid unnecessary work

- Using namedtuple for Statement impl

- Removing unneeded imports in __init__.py

- Removing automodules in PyOpenWorm docstring -- apidoc will handle these
- Adding an excepthook to filter out import_override frames

- Making DataObjectTest backwards compatible with 2.7

- Upgrading numpydoc and updating documentation

- Enabling "related topics" in docs html sidebar

- Enabling warnings

- Adding arg for ContextualizableList decontextualize init

- Changing to "pow.conf" for default config file name. Adding add_graph test

- Putting imports in a named context

- Adding base directory for translation in insert_worm.py: Makes things
  more portable

.gitignore

@@ -43,3 +43,7 @@ snakefood
 .cache
 .coverage
 .pytest_cache/*
+
+# api docs are generated but housed in the docs/ directory with the handwritten
+# kind
+docs/api

OpenWormData/scripts/insert_worm.py

@@ -48,9 +48,9 @@
 ADDITIONAL_EXPR_DATA_DIR = 'expression_data'
 
 
-def aux_data(s):
+def aux_data():
     d = get_library_location('OpenWormData')
-    return os.path.join(d, 'aux_data', s)
+    return os.path.join(d, 'aux_data')
 
 
 # TODO: Make PersonDataTranslators and Document/Website DataSource for the
@@ -61,46 +61,47 @@ def init_sources():
         WormbaseTextMatchCSVDataSource(
             key='WormbaseTextMatchCSVChannelNeuronDataSource',
             cell_type=Neuron,
-            csv_file_name=aux_data(CHANNEL_NEURON_SOURCE),
+            csv_file_name=CHANNEL_NEURON_SOURCE,
             initial_cell_column=101),
         WormbaseTextMatchCSVDataSource(
             key='WormbaseTextMatchCSVChannelMuscleDataSource',
             cell_type=Muscle,
-            csv_file_name=aux_data(CHANNEL_MUSCLE_SOURCE),
+            csv_file_name=CHANNEL_MUSCLE_SOURCE,
             initial_cell_column=6),
         WormbaseIonChannelCSVDataSource(
             key='WormbaseIonChannelCSVDataSource',
-            csv_file_name=aux_data(IONCHANNEL_SOURCE)),
+            csv_file_name=IONCHANNEL_SOURCE),
         NeuronCSVDataSource(
             key='WormAtlasNeuronTypesSource',
-            csv_file_name=aux_data(NEURON_EXPRESSION_DATA_SOURCE),
-            bibtex_files=[aux_data('bibtex_files/altun2009.bib'),
-                          aux_data('bibtex_files/WormAtlas.bib')]),
+            csv_file_name=NEURON_EXPRESSION_DATA_SOURCE,
+            bibtex_files=['bibtex_files/altun2009.bib',
+                          'bibtex_files/WormAtlas.bib']),
         WormBaseCSVDataSource(
             key='WormBaseCSVDataSource',
-            csv_file_name=aux_data(CELL_NAMES_SOURCE),
+            csv_file_name=CELL_NAMES_SOURCE,
             description="CSV converted from this Google Spreadsheet: "
                         "https://docs.google.com/spreadsheets/d/"
                         "1NDx9LRF_B2phR5w4HlEtxJzxx1ZIPT2gA0ZmNmozjos/edit#gid=1"),
         WormAtlasCellListDataSource(
             key='WormAtlasCellList',
-            csv_file_name=aux_data(LINEAGE_LIST_LOC),
+            csv_file_name=LINEAGE_LIST_LOC,
             description="CSV converted from this Google Spreadsheet: "
                         "https://docs.google.com/spreadsheets/d/"
                         "1Jc9pOJAce8DdcgkTgkUXafhsBQdrer2Y47zrHsxlqWg/edit"),
         ConnectomeCSVDataSource(
             key='EmmonsConnectomeCSVDataSource',
-            csv_file_name=aux_data(CONNECTOME_SOURCE))
+            csv_file_name=CONNECTOME_SOURCE)
     ]
 
 
-def init_extra_sources():
+def init_extra_sources(basedir):
     res = []
-    for root, _, filenames in os.walk(aux_data(ADDITIONAL_EXPR_DATA_DIR)):
+    for root, _, filenames in os.walk(os.path.join(basedir, ADDITIONAL_EXPR_DATA_DIR)):
         for filename in sorted(filenames):
             if filename.lower().endswith('.csv'):
                 name = 'NeuronCSVExpressionDataSource_' + os.path.basename(filename).rsplit('.', 1)[0]
-                res.append(NeuronCSVDataSource(csv_file_name=os.path.join(root, filename), key=name))
+                relpath = os.path.relpath(os.path.join(root, filename), basedir)
+                res.append(NeuronCSVDataSource(csv_file_name=relpath, key=name))
 
     return res
 
@@ -192,24 +193,29 @@ def infer():
     print ("filled in with inferred data")
 
 
-def do_insert(ident, config="default.conf", logging=False):
-
-    CTX = Context(ident=ident + '-data', imported=(P.CONTEXT,))
-
-    EVCTX = Context(ident=ident + '-evidence', imported=(P.CONTEXT,))
-
-    IWCTX = Context(ident=ident, imported=(CTX, EVCTX))
+def do_insert(ident, config="default.conf", logging=False, imports_context_ident=None, basedir=aux_data()):
 
     sources = init_sources()
-    extras = init_extra_sources()
+    extras = init_extra_sources(basedir)
     data_sources_by_key = {x.key: x for x in sources + extras}
     trans_map = init_translators() + init_extra_neuron_data_translators(extras)
     P.connect(configFile=config, do_logging=logging)
+    P.config()
+
+    CTX = Context(ident=ident + '-data', imported=(P.CONTEXT,), conf=P.config())
+
+    EVCTX = Context(ident=ident + '-evidence', imported=(P.CONTEXT,), conf=P.config())
+
+    IWCTX = Context(ident=ident, imported=(CTX, EVCTX), conf=P.config())
+
+    imports_context = Context(ident=imports_context_ident, conf=P.config())
+
     try:
         t0 = time()
         translators = dict()
         remaining = list(trans_map)
         last_remaining = None
+        saved_contexts = set([])
         while remaining != last_remaining:
             next_remaining = []
             for t in remaining:
@@ -234,17 +240,21 @@ def do_insert(ident, config="default.conf", logging=False):
 
                 print('\n'.join('Input({}/{}): {}'.format(i + 1, len(sources), s) for i, s in enumerate(sources)))
                 print('Translating with {}'.format(translator))
-                res = translator(*sources, output_key=output_key)
+                orig_wd = os.getcwd()
+                os.chdir(basedir)
+                try:
+                    res = translator(*sources, output_key=output_key)
+                finally:
+                    os.chdir(orig_wd)
 
                 print('Result: {}'.format(res))
                 if isinstance(res, DataWithEvidenceDataSource):
-                    res.data_context.save_context(inline_imports=True)
-                    res.data_context.save_imports()
-                    res.evidence_context.save_context(inline_imports=True)
-                    res.evidence_context.save_imports()
+                    res.data_context.save_context(inline_imports=True, saved_contexts=saved_contexts)
+                    res.data_context.save_imports(imports_context)
+                    res.evidence_context.save_context(inline_imports=True, saved_contexts=saved_contexts)
+                    res.evidence_context.save_imports(imports_context)
                     for ctx in res.contexts:
-                        ctx.save_context(inline_imports=True)
-                        ctx.save_imports()
+                        raise Exception()
 
                 if res:
                     if res.key:
@@ -259,7 +269,7 @@ def do_insert(ident, config="default.conf", logging=False):
         # attach_neuromlfiles_to_channel()
 
         t1 = time()
-        print("Saving %d objects..." % IWCTX.size())
+        print("Saving data...")
         graph = P.config('rdf.graph')
         for src in data_sources_by_key.values():
             if isinstance(src, DataWithEvidenceDataSource):
@@ -268,11 +278,10 @@ def do_insert(ident, config="default.conf", logging=False):
                 EVCTX.add_import(src.evidence_context)
                 for ctx in src.contexts:
                     IWCTX.add_import(ctx)
-        IWCTX.save_context(graph, inline_imports=True)
-        IWCTX.save_imports(graph)
-
-        print("Saved %d objects." % IWCTX.defcnt)
-        print("Saved %d triples." % IWCTX.tripcnt)
+        IWCTX.save_context(graph, saved_contexts=saved_contexts)
+        IWCTX.save_imports(imports_context)
+        print('imports context size', len(imports_context))
+        print("Saved %d triples." % IWCTX.triples_saved)
         t2 = time()
 
         print("Serializing...")

PyOpenWorm/__init__.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 
 """
+.. _pow_module:
+
 PyOpenWorm
 ==========
 
@@ -38,23 +40,6 @@
   objects are created by querying the database; these may be made out-of-date in that case.
 
 - ``a : {x_0,...,x_n}`` means ``a`` could have the value of any one of ``x_0`` through ``x_n``
-
-Classes
--------
-
-.. automodule:: PyOpenWorm.experiment
-.. automodule:: PyOpenWorm.channel
-.. automodule:: PyOpenWorm.evidence
-.. automodule:: PyOpenWorm.network
-.. automodule:: PyOpenWorm.neuron
-.. automodule:: PyOpenWorm.worm
-.. automodule:: PyOpenWorm.muscle
-.. automodule:: PyOpenWorm.connection
-.. automodule:: PyOpenWorm.relationship
-.. automodule:: PyOpenWorm.dataObject
-.. automodule:: PyOpenWorm.data
-.. automodule:: PyOpenWorm.cell
-.. automodule:: PyOpenWorm.configure
 """
 
 from __future__ import print_function
@@ -69,8 +54,7 @@
 # The c extensions are incompatible with our code...
 os.environ['WRAPT_DISABLE_EXTENSIONS'] = '1'
 
-# For re-export
-from .configure import Configure, Configureable, ConfigValue, BadConf
+from .configure import Configureable
 from .context import Context
 import yarom
 from yarom.mapper import Mapper
@@ -84,10 +68,6 @@
     "disconnect",
     "connect",
     "config",
-    "Configure",
-    "Configureable",
-    "ConfigValue",
-    "BadConf",
     ]
 
 # Base class names is empty because we won't be adding any objects to the
@@ -128,7 +108,9 @@
                   mapper=mapper)
 
 yarom.MAPPER = CONTEXT.mapper
-Overrider(yarom.MAPPER).wrap_import()
+overrider = Overrider(yarom.MAPPER)
+overrider.wrap_import()
+overrider.install_excepthook()
 
 
 def get_data(path):

PyOpenWorm/cell.py

@@ -3,9 +3,10 @@
 from string import Template
 import neuroml
 
-from PyOpenWorm.dataObject import InverseProperty
-from PyOpenWorm.channel import Channel
-from PyOpenWorm.biology import BiologyType
+from .channel import Channel
+from .biology import BiologyType
+from .dataObject import DatatypeProperty, ObjectProperty, This
+from .cell_common import CELL_RDF_TYPE
 
 __all__ = ["Cell"]
 
@@ -58,59 +59,60 @@ def _dict_merge(d1, d2):
 
 
 class Cell(BiologyType):
+
     """
     A biological cell.
 
     All cells with the same name are considered to be the same object.
 
     Parameters
     -----------
-    name : string
+    name : str
         The name of the cell
-    lineageName : string
+    lineageName : str
         The lineageName of the cell
-        Example::
+    """
 
-            >>> c = Cell(name="ADAL")
-            >>> c.lineageName() # Returns ["AB plapaaaapp"]
+    class_context = BiologyType.class_context
 
-    Attributes
-    ----------
-    name : DatatypeProperty
-        The 'adult' name of the cell typically used by biologists when discussing C. elegans
-    lineageName : DatatypeProperty
-        The lineageName of the cell
+    rdf_type = CELL_RDF_TYPE
 
-    description : DatatypeProperty
-        A description of the cell
-    divisionVolume : DatatypeProperty
-        When called with no argument, return the volume of the cell at division
-        during development.
+    divisionVolume = DatatypeProperty()
+    ''' The volume of the cell at division
 
-        When called with an argument, set the volume of the cell at division
         Example::
 
             >>> v = Quantity("600","(um)^3")
             >>> c = Cell(lineageName="AB plapaaaap")
             >>> c.divisionVolume(v)
-    """
+    '''
 
-    class_context = BiologyType.class_context
+    name = DatatypeProperty()
+    ''' The 'adult' name of the cell typically used by biologists when discussing C. elegans '''
+    wormbaseID = DatatypeProperty()
 
-    def __init__(self, name=False, lineageName=False, **kwargs):
-        super(Cell, self).__init__(**kwargs)
+    description = DatatypeProperty()
+    ''' A description of the cell '''
+
+    channel = ObjectProperty(value_type=Channel,
+                             multiple=True,
+                             inverse_of=(Channel, 'appearsIn'))
+
+    lineageName = DatatypeProperty()
+    ''' The lineageName of the cell
+        Example::
 
-        Cell.DatatypeProperty('lineageName', owner=self)
-        Cell.DatatypeProperty('name', owner=self)
-        Cell.DatatypeProperty('divisionVolume', owner=self)
-        Cell.DatatypeProperty('description', owner=self)
-        Cell.DatatypeProperty('wormbaseID', owner=self)
-        Cell.DatatypeProperty('synonym', owner=self, multiple=True)
-        Cell.ObjectProperty('channel', owner=self, multiple=True,
-                            value_type=Channel)
-        self.daughterOf = Cell.ObjectProperty(value_type=Cell)
-        self.parentOf = Cell.ObjectProperty(value_type=Cell, multiple=True)
+            >>> c = Cell(name="ADAL")
+            >>> c.lineageName() # Returns ["AB plapaaaapp"]
+    '''
 
+    synonym = DatatypeProperty(multiple=True)
+    daughterOf = ObjectProperty(value_type=This,
+                                inverse_of=(This, 'parentOf'))
+    parentOf = ObjectProperty(value_type=This, multiple=True)
+
+    def __init__(self, name=None, lineageName=None, **kwargs):
+        super(Cell, self).__init__(**kwargs)
         if name:
             self.name(name)
 
@@ -201,6 +203,4 @@ def identifier_augment(self, *args, **kwargs):
         return self.make_identifier_direct(str(self.name.defined_values[0].identifier))
 
 
-InverseProperty(Cell, 'channel', Channel, 'appearsIn')
-InverseProperty(Cell, 'daughterOf', Cell, 'parentOf')
 __yarom_mapped_classes__ = (Cell,)

PyOpenWorm/cell_common.py

@@ -0,0 +1,3 @@
+from .dataObject import BaseDataObject
+
+CELL_RDF_TYPE = BaseDataObject.base_namespace.Cell