Merge pull request #468 from isl-org/dev_to_master_0.15

dev to master for 0.15

.github/workflows/style.yml

@@ -3,8 +3,7 @@ name: Style check
 on:
   workflow_dispatch:
   push:
-    branches:
-      - master
+    branches: [master, dev]
   pull_request:
     types: [opened, reopened, synchronize]
 

.github/workflows/ubuntu.yml

@@ -3,8 +3,7 @@ name: Ubuntu CI
 on:
   workflow_dispatch:
   push:
-    branches:
-      - master
+    branches: [master, dev]
   pull_request:
     types: [opened, reopened, synchronize]
 

CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.15
+* Add support for Images and their visualization (#366)
+* Add RandLANet MIT licensed implementation (#454)
+* Refactor config parsing (#460)
+* Update pl3d, s3dis, scannet and semantic3d datasets to BaseDataSplit (#451)
+* Fix missing axis_align_matrix in Scannet preprocesing (#456)
+* Fix OpenVINOModel in Torch pipeline (#447)
+* Intel OpenVINO backend (#406)
+* Downgrade torch on Mac to 1.8.1 (#436) (#439)
+* Add bounding boxes to Visualizer DataModel (#431)
+
 ## 0.14
 * Added CHANGELOG
 * Added RandLA-Net's License (PR #423)

README.md

@@ -261,7 +261,7 @@ for details.
 
 ### Using predefined scripts
 
-[`scripts/run_pipeline.py`](https://github.com/isl-org/Open3D-ML/tree/master/scripts/run_pipeline.py)
+[`scripts/run_pipeline.py`](https://github.com/isl-org/Open3D-ML/blob/master/scripts/run_pipeline.py)
 provides an easy interface for training and evaluating a model on a dataset. It saves
 the trouble of defining specific model and passing exact configuration.
 
@@ -322,17 +322,16 @@ The table shows the available models and datasets for the segmentation task and
 
 | Model / Dataset    | SemanticKITTI | Toronto 3D | S3DIS | Semantic3D | Paris-Lille3D | ScanNet |
 |--------------------|---------------|----------- |-------|--------------|-------------|---------|
-| RandLA-Net (tf)    | [53.7](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantickitti_202010091306.zip) |  [69.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_toronto3d_202010091250.zip) |  [67.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_s3dis_202106011448utc.zip)    | [76.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantic3d_202012120312utc.zip) |  [70.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_parislille3d_202012160654utc.zip) | - |
-| RandLA-Net (torch) | [52.8](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantickitti_202009090354utc.pth)        |     [71.2](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_toronto3d_202010091306utc.pth)  |  [67.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_s3dis_202010091238.pth)  | [76.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantic3d_202012120312utc.pth) |  [70.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_parislille3d_202012160654utc.pth) | - |
+| RandLA-Net (tf)    | [53.7](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantickitti_202201071330utc.zip) |  [73.7](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_toronto3d_202201071330utc.zip) |  [70.9](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_s3dis_202201071330utc.zip)    | [76.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantic3d_202201071330utc.zip) |  [70.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_parislille3d_202201071330utc.zip)* | - |
+| RandLA-Net (torch) | [52.8](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantickitti_202201071330utc.pth)        |     [74.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_toronto3d_202201071330utc.pth)  |  [70.9](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_s3dis_202201071330utc.pth)  | [76.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_semantic3d_202201071330utc.pth) |  [70.0](https://storage.googleapis.com/open3d-releases/model-zoo/randlanet_parislille3d_202201071330utc.pth)* | - |
 | KPConv     (tf)    | [58.7](https://storage.googleapis.com/open3d-releases/model-zoo/kpconv_semantickitti_202010021102utc.zip)         |     [65.6](https://storage.googleapis.com/open3d-releases/model-zoo/kpconv_toronto3d_202012221551utc.zip)  |  [65.0](https://storage.googleapis.com/open3d-releases/model-zoo/kpconv_s3dis_202010091238.zip) | - |  [76.7](https://storage.googleapis.com/open3d-releases/model-zoo/kpconv_parislille3d_202011241550utc.zip) | - |
 | KPConv     (torch) | [58.0](https://storage.googleapis.com/open3d-releases/model-zoo/kpconv_semantickitti_202009090354utc.pth)          |     [65.6](https://storage.googleapis.com/open3d-releases/model-zoo/kpconv_toronto3d_202012221551utc.pth) |  [60.0](https://storage.googleapis.com/open3d-releases/model-zoo/kpconv_s3dis_202010091238.pth)  | - | [76.7](https://storage.googleapis.com/open3d-releases/model-zoo/kpconv_parislille3d_202011241550utc.pth) | - |
 | SparseConvUnet (torch)| - | - | - | - | - | [68](https://storage.googleapis.com/open3d-releases/model-zoo/sparseconvunet_scannet_202105031316utc.pth) |
 | SparseConvUnet (tf)| - | - | - | - | - | [68.2](https://storage.googleapis.com/open3d-releases/model-zoo/sparseconvunet_scannet_202105031316utc.zip) |
 | PointTransformer (torch)| - | - | [69.2](https://storage.googleapis.com/open3d-releases/model-zoo/pointtransformer_s3dis_202109241350utc.pth) | - | - | - |
 | PointTransformer (tf)| - | - | [69.2](https://storage.googleapis.com/open3d-releases/model-zoo/pointtransformer_s3dis_202109241350utc.zip) | - | - | - |
 
-
-> **_NOTE:_**  RandLA-Net comes with a [Creative Commons BY-NC-SA 4.0](https://github.com/QingyongHu/RandLA-Net/blob/master/LICENSE) license which prohibits commercial use. We are working on a new implementation which will be available under the standard Open3D license (MIT).
+(*) Using weights from original author.
 
 ### Object Detection
 
@@ -383,11 +382,11 @@ The following is a list of datasets for which we provide dataset reader classes.
 * SemanticKITTI ([project page](http://semantic-kitti.org/))
 * Toronto 3D ([github](https://github.com/WeikaiTan/Toronto-3D))
 * Semantic 3D ([project-page](http://www.semantic3d.net/))
-* S3DIS ([project-page](http://3dsemantics.stanford.edu/))
+* S3DIS ([project-page](http://buildingparser.stanford.edu/dataset.html))
 * Paris-Lille 3D ([project-page](https://npm3d.fr/paris-lille-3d))
 * Argoverse ([project-page](https://www.argoverse.org/))
 * KITTI ([project-page](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d))
-* Lyft ([project-page](https://self-driving.lyft.com/level5/data/))
+* Lyft ([project-page](https://level-5.global/data))
 * nuScenes ([project-page](https://www.nuscenes.org/))
 * Waymo ([project-page](https://waymo.com/open/))
 * ScanNet([project-page](http://www.scan-net.org/))
@@ -404,6 +403,7 @@ For downloading these datasets visit the respective webpages and have a look at
 * [Adding a new model](docs/howtos.md#adding-a-new-model)
 * [Adding a new dataset](docs/howtos.md#adding-a-new-dataset)
 * [Visualize and compare input data, ground truth and results in TensorBoard](docs/tensorboard.md)
+* [Inference with Intel OpenVINO](docs/openvino.md)
 
 ## Contribute
 There are many ways to contribute to this project. You can:
@@ -425,7 +425,7 @@ Please also check out our communication channels to get in contact with the comm
 
 <!--* [GitHub Issue](https://github.com/isl-org/Open3D/issues): bug reports, feature requests, etc.-->
 * [Forum](https://github.com/isl-org/Open3D/discussions): discussion on the usage of Open3D.
-* [Discord Chat](https://discord.gg/D35BGvn): online chats, discussions,
+* [Discord Chat](https://discord.com/invite/D35BGvn): online chats, discussions,
   and collaboration with other users and developers.
 
 ## Citation

ci/run_ci.sh

@@ -31,6 +31,7 @@ python -m pip install -U torch==${TORCH_GLNX_VER} -f https://download.pytorch.or
 python -m pip install -U pytest=="$PYTEST_VER" \
     pytest-randomly=="$PYTEST_RANDOMLY_VER"
 python -m pip install -U yapf=="$YAPF_VER"
+python -m pip install -U openvino-dev==2021.4.2
 
 echo 3. Configure for bundling the Open3D-ML part
 echo

docs/openvino.md

@@ -0,0 +1,40 @@
+# OpenVINO backend
+
+Open3D-ML allows to use [Intel
+OpenVINO](https://github.com/openvinotoolkit/openvino) as an optional backend for deep learning models inference.
+
+## Install
+
+Install a compatible version of OpenVINO with:
+
+```sh
+pip install -r requirements-openvino.txt
+```
+
+## Usage
+
+To enable OpenVINO, wrap a model in `ml3d.models.OpenVINOModel` class. In example,
+
+```python
+net = ml3d.models.PointPillars(**cfg.model, device='cpu')
+net = ml3d.models.OpenVINOModel(net)
+```
+
+Then use `net` as usual.
+
+## Supported hardware
+
+OpenVINO supports Intel CPUs, GPUs and VPUs. By default, model is executed on CPU.
+To switch between devices, use `net.to` option:
+
+```python
+net.to("cpu")     # CPU device (default)
+net.to("gpu")     # GPU device
+net.to("myriad")  # VPU device
+```
+
+## Supported models
+
+* `RandLA-Net` (tf, torch)
+* `KPConv` (tf, torch)
+* `PointPillars` (torch)

ml3d/configs/randlanet_parislille3d.yml

@@ -19,48 +19,30 @@ model:
   name: RandLANet
   batcher: DefaultBatcher
   ckpt_path: # path/to/your/checkpoint
-  dim_feature: 8
-  dim_input: 3
-  dim_output:
-  - 16
-  - 64
-  - 128
-  - 256
-  - 512
-  grid_size: 0.08
-  ignored_label_inds:
-  - 0
-  k_n: 16
-  num_classes: 9
+  num_neighbors: 16
   num_layers: 5
   num_points: 65536
-  sub_sampling_ratio:
-  - 4
-  - 4
-  - 4
-  - 4
-  - 2
+  num_classes: 9
+  ignored_label_inds: [0]
+  sub_sampling_ratio: [4, 4, 4, 4, 2]
+  in_channels: 3
+  dim_features: 8
+  dim_output: [16, 64, 128, 256, 512]
+  grid_size: 0.06
   weight_decay: 0.001
-  t_align: true
-  t_normalize:
-    recentering: [0, 1]
-  t_augment:
-    turn_on: false
-    rotation_method: vertical
-    scale_anisotropic: false
-    symmetries: true
-    noise_level: 0.01
-    min_s: 0.9
-    max_s: 1.1
+  augment:
+    recenter:
+      dim: [0, 1, 2]
 pipeline:
   name: SemanticSegmentation
-  adam_lr: 0.004
+  optimizer:
+    lr: 0.001
   batch_size: 2
   main_log_dir: ./logs
-  max_epoch: 200
+  max_epoch: 100
   save_ckpt_freq: 5
   scheduler_gamma: 0.9886
-  test_batch_size: 2
+  test_batch_size: 1
   train_sum_dir: train_log
   val_batch_size: 2
   summary:

ml3d/configs/randlanet_s3dis.yml

@@ -13,40 +13,35 @@ model:
   name: RandLANet
   batcher: DefaultBatcher
   ckpt_path: # path/to/your/checkpoint
-  dim_feature: 8
-  dim_input: 6
-  dim_output:
-  - 16
-  - 64
-  - 128
-  - 256
-  - 512
-  grid_size: 0.04
-  ignored_label_inds: []
-  k_n: 16
-  num_classes: 13
+  num_neighbors: 16
   num_layers: 5
   num_points: 40960
-  sub_sampling_ratio:
-  - 4
-  - 4
-  - 4
-  - 4
-  - 2
-  t_normalize:
-    method: linear
-    normalize_points: False
-    feat_bias: 0
-    feat_scale: 1
+  num_classes: 13
+  ignored_label_inds: []
+  sub_sampling_ratio: [4, 4, 4, 4, 2]
+  in_channels: 6
+  dim_features: 8
+  dim_output: [16, 64, 128, 256, 512]
+  grid_size: 0.04
+  augment:
+    recenter:
+      dim: [0, 1]
+    rotate:
+      method: vertical
+    scale:
+      min_s: 0.9
+      max_s: 1.1
+    noise:
+      noise_std: 0.001
 pipeline:
   name: SemanticSegmentation
-  adam_lr: 0.01
+  optimizer:
+    lr: 0.001
   batch_size: 2
-  learning_rate: 0.01
   main_log_dir: ./logs
-  max_epoch: 100
-  save_ckpt_freq: 20
-  scheduler_gamma: 0.95
+  max_epoch: 200
+  save_ckpt_freq: 3
+  scheduler_gamma: 0.99
   test_batch_size: 3
   train_sum_dir: train_log
   val_batch_size: 2

ml3d/configs/randlanet_semantic3d.yml

@@ -3,8 +3,7 @@ dataset:
   dataset_path: # path/to/your/dataset
   cache_dir: ./logs/cache_small3d/
   class_weights: [5181602, 5012952, 6830086, 1311528, 10476365, 946982, 334860, 269353]
-  ignored_label_inds:
-  - 0
+  ignored_label_inds: [0]
   num_points: 65536
   test_result_folder: ./test
   use_cache: true
@@ -17,52 +16,43 @@ model:
   name: RandLANet
   batcher: DefaultBatcher
   ckpt_path: # path/to/your/checkpoint
-  dim_feature: 8
-  dim_input: 6
-  dim_output:
-  - 16
-  - 64
-  - 128
-  - 256
-  - 512
-  grid_size: 0.06
-  ignored_label_inds:
-  - 0
-  k_n: 16
-  num_classes: 8
+  num_neighbors: 16
   num_layers: 5
   num_points: 65536
-  sub_sampling_ratio:
-  - 4
-  - 4
-  - 4
-  - 4
-  - 2
-  t_normalize:
-    method: linear
-    feat_bias: 0
-    feat_scale: 255
-  t_augment:
-    turn_on: true
-    rotation_method: vertical
-    scale_anisotropic: true
-    symmetries: [true, false, false]
-    noise_level: 0.001
-    min_s: 0.8
-    max_s: 1.2
+  num_classes: 8
+  ignored_label_inds: [0]
+  sub_sampling_ratio: [4, 4, 4, 4, 2]
+  in_channels: 6
+  dim_features: 8
+  dim_output: [16, 64, 128, 256, 512]
+  grid_size: 0.06
+  augment:
+    recenter:
+      dim: [0, 1]
+    normalize:
+      feat:
+        method: linear
+        bias: 0
+        scale: 255
+    rotate:
+      method: vertical
+    scale:
+      min_s: 0.9
+      max_s: 1.1
+    noise:
+      noise_std: 0.001
 pipeline:
   name: SemanticSegmentation
-  adam_lr: 0.01
+  optimizer:
+    lr: 0.001
   batch_size: 2
-  learning_rate: 0.01
   main_log_dir: ./logs
   max_epoch: 100
   save_ckpt_freq: 5
-  scheduler_gamma: 0.95
+  scheduler_gamma: 0.9886
   test_batch_size: 1
   train_sum_dir: train_log
-  val_batch_size: 1
-  test_compute_metric: false
+  val_batch_size: 2
   summary:
     record_for: []
     max_pts: