consmetic changes

Cargo.toml

@@ -1,6 +1,7 @@
 [profile.release]
-# debug = true
+debug = false
 lto = "thin"
+opt-level = "s"
 
 [workspace]
 

editor-support/vscode/syntaxes/lopez-crawl-directive.rion

@@ -263,24 +263,32 @@ keywords : context {
 
 ruleset_namespace : context {
    : pattern {
-      regex \= (first|collect|count|sum)
+      regex \= (first|collect|distinct|count|sum|group)
       styles[] = .ruleset_aggregator;
    }
    : pattern {
-      regex \= (name|text|html|inner-html|attr)
+      regex \= (name|text|html|inner-html|attrs?|classes|id|parent|children|select-any|select-all)
       styles[] = .ruleset_extractor;
    }
    : pattern {
-      regex \= (is-null|is-not-null|length|hash|get|flatten|each|capture|all-captures)
+      regex \= (is-null|is-not-null|hash|not|as-number|greater-than|lesser-than|equals|length|is-empty|get|flatten|each|filter|pretty|capture|all-captures|matches|replace|with)
       styles[] = .ruleset_transformer;
    }
+   : pattern {
+      regex \= (\!explode)
+      styles[] = .keyword;
+   }
 }
 
 numeric : context {
    : pattern {
-      regex          \= (\b\d+)
+      regex          \= (\b[0-9_]+e?[+-]\d+)
       styles []       = .numeric;
    }
+   : pattern {
+      regex          \= (true|false)
+      styles[]        = .keyword;
+   }
 }
 
 }

editor-support/vscode/syntaxes/lopez-crawl-directives.tmLanguage.plist

@@ -268,10 +268,6 @@
           <key>include</key>
           <string>#string</string>
         </dict>
-        <dict>
-          <key>include</key>
-          <string>#numeric</string>
-        </dict>
        </array>
     </dict>
     <key>main__4</key>
@@ -286,10 +282,16 @@
       <array> 
         <dict>
           <key>match</key>
-          <string>(\b\d+)</string>
+          <string>(\b[0-9_]+e?[\x{002b}-]\d+)</string>
           <key>name</key>
           <string>constant.numeric.lcd</string>
         </dict>
+        <dict>
+          <key>match</key>
+          <string>(true|false)</string>
+          <key>name</key>
+          <string>keyword.lcd</string>
+        </dict>
        </array>
     </dict>
     <key>punctuation</key>
@@ -318,11 +320,11 @@
           <key>match</key>
           <string>(name|text|html|inner-html|attrs?|classes|id|parent|children|select-any|select-all)</string>
           <key>name</key>
-          <string>variable.lcd</string>
+          <string>entity.name.function.lcd</string>
         </dict>
         <dict>
           <key>match</key>
-          <string>(is-null|is-not-null|hash|not|as-number|greater-than|lesser-than|equals|length|get|flatten|each|filter|pretty|capture|all-captures|replace)</string>
+          <string>(is-null|is-not-null|hash|not|as-number|greater-than|lesser-than|equals|length|is-empty|get|flatten|each|filter|pretty|capture|all-captures|matches|replace|with)</string>
           <key>name</key>
           <string>entity.name.function.lcd</string>
         </dict>

entalator/src/main.rs

@@ -6,15 +6,18 @@ use std::{env, fs, io};
 const LOPEZ_BIN: &[u8] = include_bytes!("../../target/release/lopez");
 const LOPEZ_LIB: Dir = include_dir::include_dir!("../std-lopez");
 
+const LIB_PATH: &str = "/usr/share/lopez/lib";
+const BIN_PATH: &str = "/usr/local/bin/lopez";
+
 fn install() -> io::Result<()> {
-    let lib_path: PathBuf = "/usr/share/lopez/lib".parse().expect("infallible");
 
-    println!("Installing `lopez` to `/usr/local/bin`");
+    println!("Installing `lopez` to `{}`", BIN_PATH);
 
-    fs::write("/usr/local/bin/lopez", LOPEZ_BIN)?;
-    fs::set_permissions("/usr/local/bin/lopez", fs::Permissions::from_mode(0o711))?;
+    fs::write(BIN_PATH, LOPEZ_BIN)?;
+    fs::set_permissions(BIN_PATH, fs::Permissions::from_mode(0o711))?;
 
-    println!("Installing `std-lopez` to `usr/share/lopez`");
+    let lib_path: PathBuf = LIB_PATH.parse().expect("infallible");
+    println!("Installing `std-lopez` to `{}`", LIB_PATH);
 
     println!("Creating folder structure");
 

lib-lopez/src/crawler/counter.rs

@@ -123,7 +123,8 @@ impl StatsTracker {
                 self.quota as usize,
             ),
             hit_rate: Human(
-                (self.already_done + self.counter.n_closed() - self.counter.n_error()
+                (self.already_done + self.counter.n_closed()
+                    - self.counter.n_error()
                     - self
                         .last
                         .as_ref()
@@ -216,3 +217,28 @@ impl Display for Stats {
         Ok(())
     }
 }
+
+struct Smoother {
+    last_state: f64,
+    last_variance: f64,
+    state_variance: f64,
+    output_variance: f64,
+}
+
+fn par(a: f64, b: f64) -> f64 {
+    a * b / (a + b)
+}
+
+impl Smoother {
+    fn smooth(&mut self, input: f64) -> f64 {
+        let variance = self.last_variance + self.state_variance;
+        let new_state = self.last_state
+            + variance / (variance + self.output_variance) * (input - self.last_state);
+        let new_variance = par(variance, self.output_variance);
+
+        self.last_state = new_state;
+        self.last_variance = new_variance;
+
+        new_state
+    }
+}

lib-lopez/src/crawler/worker.rs

@@ -397,7 +397,7 @@ impl<WF: WorkerBackendFactory> CrawlWorker<WF> {
                     .ensure_error(page_url)
                     .await
                     .map_err(|err| err.into())?;
-                
+
                 // This needs to be the last thing (because of `?`).
                 self.task_counter.register_error();
             }
@@ -407,7 +407,7 @@ impl<WF: WorkerBackendFactory> CrawlWorker<WF> {
                     .ensure_error(page_url)
                     .await
                     .map_err(|err| err.into())?;
-                
+
                 // This needs to be the last thing (because of `?`).
                 self.task_counter.register_error();
             }
@@ -480,7 +480,7 @@ impl<WF: WorkerBackendFactory> CrawlWorker<WF> {
 
                         // Register close, no matter the status.
                         worker_ref.task_counter.register_closed();
-                        
+
                         // Now, analyze results:
                         if let Err(error) = result {
                             worker_ref.task_counter.register_error();

lib-lopez/src/directives/mod.rs

@@ -1,6 +1,7 @@
 mod aggregator;
 mod extractor;
 mod parse;
+mod parse_utils;
 mod transformer;
 mod value_ext;
 mod variable;
@@ -49,13 +50,18 @@ fn load_items_from<'a, P: AsRef<Path>>(
     module_name: &str,
     paths: &'a [P],
 ) -> Result<(&'a P, Vec<Item>), String> {
+    let formatted_module_name = if module_name.is_empty() {
+        "<main>"
+    } else {
+        module_name
+    };
+
     let (path, module_str) = read_from_many(paths)
-        .map_err(|err| format!("could not open module `{}`: {}", module_name, err))?;
+        .map_err(|err| format!("could not open module `{}`: {}", formatted_module_name, err))?;
 
     let module = parse::entrypoint(&module_str)
-        .map_err(|err| format!("failed to parse `{}`: {}", module_name, err))?
-        .1
-        .map_err(|err| format!("failed to interpret `{}`: {}", module_name, err))?;
+        .map_err(|err| format!("failed to parse `{}`: {}", formatted_module_name, err))?
+        .map_err(|err| format!("failed to interpret `{}`: {}", formatted_module_name, err))?;
 
     Ok((path, module))
 }
@@ -295,70 +301,73 @@ impl Directives {
         let duplicates = self.find_duplicate_rules();
         if !duplicates.is_empty() {
             issues.push(format!(
-                "There are duplicated rules in directives: \n\t- {}",
-                duplicates.into_iter().collect::<Vec<_>>().join("\n\t- ")
+                "There are duplicated rules in directives: \n    {}",
+                duplicates.into_iter().collect::<Vec<_>>().join("\n    ")
             ));
         }
 
         let invalid_seeds = self.find_invalid_seeds();
         if !invalid_seeds.is_empty() {
             issues.push(format!(
-                "There are seeds on the frontier or outside your boundaries: \n\t- {}",
+                "There are seeds on the frontier or outside your boundaries: \n    {}",
                 invalid_seeds
                     .into_iter()
                     .map(|url| url.as_str().to_owned())
                     .collect::<Vec<_>>()
-                    .join("\n\nt- ")
+                    .join("\n    ")
             ));
         }
 
         let invalid = self.find_invalid_set_variables();
         if !invalid.is_empty() {
             issues.push(format!(
                 "There are invalid set-variable definitions \
-                (these name are not known): \n\t- {}",
-                invalid.into_iter().collect::<Vec<_>>().join("\n\t- "),
+                (these name are not known): \n    {}",
+                invalid.into_iter().collect::<Vec<_>>().join("\n    "),
             ));
         }
 
         let duplicates = self.find_duplicate_set_variables();
         if !duplicates.is_empty() {
             issues.push(format!(
                 "There are duplicate set-variable definitions \
-                (these definitions are global): \n\t- {}",
-                duplicates.into_iter().collect::<Vec<_>>().join("\n\t- "),
+                (these definitions are global): \n    {}",
+                duplicates.into_iter().collect::<Vec<_>>().join("\n   "),
             ));
         }
 
         let bad_values = self.find_bad_set_variable_values();
         if !bad_values.is_empty() {
             issues.push(format!(
-                "There are bad values for set-variables: \n\t- {}",
+                "There are bad values for set-variables: \n    {}",
                 bad_values
                     .into_iter()
                     .map(|err| err.to_string())
                     .collect::<Vec<_>>()
-                    .join("\n\nt- "),
+                    .join("\n    "),
             ))
         }
 
         let type_errors = self.find_type_errors();
         if !type_errors.is_empty() {
             issues.push(format!(
-                "There are type errors for these rules: \n\t- {}",
+                "There are type errors for these rules: \n    {}",
                 type_errors
                     .into_iter()
                     .map(|(name, err)| format!("{}: {}", name, err))
                     .collect::<Vec<_>>()
-                    .join("\n\t- ")
+                    .join("\n    ")
             ))
         }
 
         if !issues.is_empty() {
-            return Err(issues.join("\n"));
+            Err(format!(
+                "There are issues with your configuration: \n{}",
+                issues.join("\n")
+            ))
+        } else {
+            Ok(())
         }
-
-        Ok(())
     }
 
     /// Loads directives from a given file while also loading all dependencies.

lib-lopez/src/directives/parse.rs

@@ -14,6 +14,7 @@ use std::str::FromStr;
 use url::Url;
 
 use super::*;
+use super::parse_utils::ParseError;
 
 /// Defines end of file (lol!):
 fn eof(i: &str) -> IResult<&str, ()> {
@@ -839,6 +840,8 @@ fn boundary_test() {
 fn literal(i: &str) -> IResult<&str, Value> {
     alt((
         map(escaped_string, Value::String),
+        map(tag("true"), |_| true.into()),
+        map(tag("false"), |_| false.into()),
         map_res(tuple((digit1, not(tag(".")))), |(number, _): (&str, ())| {
             number.parse::<u64>().map(|num| num.into())
         }),
@@ -967,11 +970,11 @@ fn item_test() {
     //     ));
 }
 
-pub fn entrypoint(i: &str) -> IResult<&str, Result<Vec<Item>, String>> {
-    all_consuming(map(
+pub fn entrypoint(i: &str) -> Result<Result<Vec<Item>, String>, ParseError> {
+    ParseError::map_iresult(i, all_consuming(map(
         tuple((whitespace, many0(trailing_whitespace(item)))),
         |(_, results)| results.into_iter().collect::<Result<Vec<_>, _>>(),
-    ))(i)
+    ))(i))
 }
 
 #[test]
@@ -980,6 +983,5 @@ fn entrypoint_test() {
         "select * { } set foo = \"bar\"; allow \"foo\";\n"
     ))
     .unwrap()
-    .1
     .unwrap();
 }