refactor: split scanner modules

Cargo.toml

@@ -15,4 +15,4 @@ path = "scie-grammar"
 members = [
  "scie-grammar",
  "scie-scanner"
-]
\ No newline at end of file
+]

scie-grammar/Cargo.toml

@@ -7,9 +7,6 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-# regex library
-onig = "6"
-
 # json convert
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

scie-grammar/src/grammar/grammar.rs

 use std::collections::BTreeMap as Map;
 
-use onig::*;
-
 use crate::grammar::line_tokens::{LineTokens, TokenTypeMatcher};
 use crate::grammar::{ScopeListElement, StackElement};
 use crate::inter::{IRawGrammar, IRawRepository, IRawRepositoryMap, IRawRule};
@@ -86,13 +84,6 @@ impl Grammar {
             _token_type_matchers: vec![],
         }
     }
-    // todo: refactor to callback ??
-    pub fn create_onig_scanner(&self, sources: String) -> Regex {
-        Regex::new(sources.as_str()).unwrap()
-    }
-    pub fn create_onig_string(&self, sources: String) -> String {
-        sources
-    }
 
     fn tokenize(
         &mut self,
@@ -231,21 +222,6 @@ impl Grammar {
             is_first_line,
             line_pos == anchor_position,
         );
-        // todo: refactor to scanner
-        let rules = rule_scanner.debug_reg_exps;
-        for rule in rules {
-            let regex = Regex::new(rule.as_str()).unwrap();
-            if let Some(captures) = regex.captures(line_text.as_str()) {
-                for (i, pos) in captures.iter_pos().enumerate() {
-                    match pos {
-                        Some((beg, end)) =>
-                            println!("Group {} line:{}, pos: {}:{}", i, line_pos, beg, end),
-                        None =>
-                            println!("Group {} is not captured", i)
-                    }
-                }
-            }
-        }
     }
 
     pub fn tokenize_line(&mut self, line_text: String, prev_state: Option<StackElement>) {

scie-grammar/src/lib.rs

 // todo: remove after finish
 #![allow(dead_code)]
 
-extern crate onig;
-
 #[macro_use]
 extern crate serde_derive;
 
@@ -13,4 +11,3 @@ pub mod grammar;
 pub mod inter;
 pub mod matcher;
 pub mod rule;
-pub mod scanner;

scie-scanner/Cargo.toml

@@ -7,3 +7,5 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+# regex library
+onig = "6"

scie-scanner/src/lib.rs

-#[cfg(test)]
-mod tests {
-    #[test]
-    fn it_works() {
-        assert_eq!(2 + 2, 4);
-    }
-}
+pub mod scanner;

scie-grammar/src/scanner/scanner.rs → scie-scanner/src/scanner/scanner.rs

-use onig::Regex;
+use onig::{Regex, EncodedBytes, SearchOptions, MatchParam, Error};
 
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone)]
 pub struct IOnigCaptureIndex {
     pub start: usize,
     pub end: usize,
@@ -14,7 +14,7 @@ pub struct IOnigMatch {
     pub capture_indices: Vec<IOnigCaptureIndex>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone)]
 pub struct Scanner {
     pub index: usize,
     pub patterns: Vec<String>,