From 43f606106d1c034d4a410347c3862fcda6af5a02 Mon Sep 17 00:00:00 2001
From: Alan Knowles <alan@akbkhome.com>
Date: Fri, 23 Sep 2011 12:01:30 +0800
Subject: [PATCH] Pman.SearchTokenizer.js

---
 Pman.SearchTokenizer.js | 133 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 133 insertions(+)

diff --git a/Pman.SearchTokenizer.js b/Pman.SearchTokenizer.js
index e69de29b..a55605de 100644
--- a/Pman.SearchTokenizer.js
+++ b/Pman.SearchTokenizer.js
@@ -0,0 +1,133 @@
+/**
+ *
+ * Search tokenizer - used with fields that are sent to Text_SearchParser
+ * 
+ *
+ * usage :
+ *  x = new Pman.SearchTokenizer('a and b or "test this" or ( tst:aval and vvv:erer }')
+ *  data = x.parse();
+ *
+ *
+ *  or
+ *  data = Pman.SearchTokenizer.parse(....)
+ */
+ 
+Pman.SearchTokenizer = function(s)
+{
+        this.str = s;
+        this.strlen = s.length;
+        this.i = 0;
+        this.tokens = [];
+       //print_r(this);
+}
+Pman.SearchTokenizer.prototype =  {
+    i : 0,
+    str : '',
+    strlen : 0,
+    tokens : false ,
+       //print_r(this);
+    
+    parse : function ()
+    {
+        var c;
+        while(true) {
+            c = this.getChar();
+            
+            if (false === c) { //eof..
+                return this.tokens;
+            }
+            switch(c) {
+                case ' ': continue;
+                case ':': this.tokens.push( { type : ':' }) ; break;
+                case '(': this.tokens.push( { type : '(' }) ; break;
+                case ')': this.tokens.push( { type : ')' }) ; break;
+                default:
+                    this.ungetChar();
+                    this.strParse();
+                    break;
+                
+               
+            }
+        }
+        // should not get here...
+        return this.tokens;
+    },
+    strParse : function ()
+    {
+        
+        str = '';
+        while(true) {
+            if (false === (c = this.getChar())) {
+                this.addStr(str);
+                return;
+            }
+            switch(c) {
+                // end chars.
+                case ' ': 
+                case ':': 
+                case '(': 
+                case ')': this.addStr(str); this.ungetChar(); return;
+                case '"': 
+                    if (strlen(str)) {
+                        this.addStr(str); 
+                        str = '';
+                    }
+                    this.strParseQuoted(c);
+                    break;
+                    
+                default : 
+                    str .= c;
+                    continue;
+            }
+            
+        }
+    }
+    function strParseQuoted(end) 
+    {
+        str = '';   /// ignore \" slashed ???
+        while(true) {
+            if (false === (c = this.getChar())) {
+                this.addStr(str,true);
+                return;
+            }
+            if (c == end) {
+                this.addStr(str,true);
+                return;
+            }
+            str .= c;
+        }
+            
+    }
+    function addStr(s, q=false) { //q == quoted..
+        s = q ? s : trim(s);
+        if (!strlen(s)) {
+            return;
+        }
+        if (!q) {
+            
+            if ((strtoupper(s) == 'AND') || (strtoupper(s) == 'OR')) {
+                this.tokens[] = new Text_SearchParser_Token_Op(strtoupper(s));
+                return;
+            }
+        }
+        this.tokens[] = new Text_SearchParser_Token_String(s);
+    }
+    
+    function getChar()
+    {
+        if (this.i >= this.strlen) {
+            return false;
+        }
+        c = this.str[this.i];
+        this.i++;
+        return c;
+    }
+    function ungetChar()
+    {
+        this.i--;
+    }
+    
+    
+    
+    
+}
\ No newline at end of file
-- 
2.39.2