[tomboy] Import Hyena.Json from Banshee.
- From: Sanford Armstrong <sharm src gnome org>
- To: svn-commits-list gnome org
- Subject: [tomboy] Import Hyena.Json from Banshee.
- Date: Mon, 18 May 2009 10:22:42 -0400 (EDT)
commit 450c7fd5b100dd19581d60fc884b6e549fa048bf
Author: Sandy Armstrong <sanfordarmstrong gmail com>
Date: Fri May 15 18:47:00 2009 -0700
Import Hyena.Json from Banshee.
---
.../WebSyncService/Hyena.Json/Deserializer.cs | 163 ++++++++++
.../WebSyncService/Hyena.Json/IJsonCollection.cs | 39 +++
.../Addins/WebSyncService/Hyena.Json/JsonArray.cs | 60 ++++
.../Addins/WebSyncService/Hyena.Json/JsonObject.cs | 60 ++++
.../Hyena.Json/Tests/DeserializerTests.cs | 102 +++++++
.../Hyena.Json/Tests/TokenizerTests.cs | 207 +++++++++++++
Tomboy/Addins/WebSyncService/Hyena.Json/Token.cs | 111 +++++++
.../Addins/WebSyncService/Hyena.Json/TokenType.cs | 51 ++++
.../Addins/WebSyncService/Hyena.Json/Tokenizer.cs | 316 ++++++++++++++++++++
9 files changed, 1109 insertions(+), 0 deletions(-)
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/Deserializer.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/Deserializer.cs
new file mode 100644
index 0000000..15afe8c
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/Deserializer.cs
@@ -0,0 +1,163 @@
+//
+// Deserializer.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+using System;
+using System.IO;
+
+namespace Hyena.Json
+{
+ public class Deserializer
+ {
+ private Tokenizer tokenizer = new Tokenizer ();
+
+ public Deserializer () { }
+ public Deserializer (string input) { SetInput (input); }
+ public Deserializer (Stream stream) { SetInput (stream); }
+ public Deserializer (StreamReader reader) { SetInput (reader); }
+
+ public Deserializer SetInput (StreamReader reader)
+ {
+ tokenizer.SetInput (reader);
+ return this;
+ }
+
+ public Deserializer SetInput (Stream stream)
+ {
+ tokenizer.SetInput (stream);
+ return this;
+ }
+
+ public Deserializer SetInput (string input)
+ {
+ tokenizer.SetInput (input);
+ return this;
+ }
+
+ public object Deserialize ()
+ {
+ Token token = CheckScan (TokenType.Value, true);
+ if (token == null) {
+ return null;
+ }
+
+ return Parse (token);
+ }
+
+ private object Parse (Token token)
+ {
+ if (token.Type == TokenType.ObjectStart) {
+ return ParseObject ();
+ } else if (token.Type == TokenType.ArrayStart) {
+ return ParseArray ();
+ }
+
+ return token.Value;
+ }
+
+ private JsonObject ParseObject ()
+ {
+ JsonObject obj = new JsonObject ();
+
+ while (true) {
+ Token key = CheckScan (TokenType.String | TokenType.ObjectFinish);
+ if (key.Type == TokenType.ObjectFinish) {
+ break;
+ }
+
+ CheckScan (TokenType.Colon);
+ Token value = CheckScan (TokenType.Value);
+
+ object value_val = value.Value;
+ if (value.Type == TokenType.ArrayStart) {
+ value_val = ParseArray ();
+ } else if (value.Type == TokenType.ObjectStart) {
+ value_val = ParseObject ();
+ }
+
+ obj.Add ((string)key.Value, value_val);
+
+ Token token = CheckScan (TokenType.Comma | TokenType.ObjectFinish);
+ if (token.Type == TokenType.ObjectFinish) {
+ break;
+ }
+ }
+
+ return obj;
+ }
+
+ private JsonArray ParseArray ()
+ {
+ JsonArray array = new JsonArray ();
+
+ while (true) {
+ Token value = CheckScan (TokenType.Value | TokenType.ArrayFinish);
+ if (value.Type == TokenType.ArrayFinish) {
+ break;
+ }
+
+ array.Add (Parse (value));
+
+ Token token = CheckScan (TokenType.Comma | TokenType.ArrayFinish);
+ if (token.Type == TokenType.ArrayFinish) {
+ break;
+ }
+ }
+
+ return array;
+ }
+
+ private Token CheckScan (TokenType expected)
+ {
+ return CheckScan (expected, false);
+ }
+
+ private Token CheckScan (TokenType expected, bool eofok)
+ {
+ Token token = tokenizer.Scan ();
+ if (token == null && eofok) {
+ return null;
+ } else if (token == null) {
+ UnexpectedEof (expected);
+ } else if ((expected & token.Type) == 0) {
+ UnexpectedToken (expected, token);
+ }
+ return token;
+ }
+
+ private void UnexpectedToken (TokenType expected, Token got)
+ {
+ throw new ApplicationException (String.Format ("Unexpected token {0} at [{1}:{2}]; expected {3}",
+ got.Type, got.SourceLine, got.SourceColumn, expected));
+ }
+
+ private void UnexpectedEof (TokenType expected)
+ {
+ throw new ApplicationException (String.Format ("Unexpected End of File; expected {0}", expected));
+ }
+ }
+}
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/IJsonCollection.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/IJsonCollection.cs
new file mode 100644
index 0000000..4d46bf7
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/IJsonCollection.cs
@@ -0,0 +1,39 @@
+//
+// IJsonCollection.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+using System;
+using System.Collections;
+
+namespace Hyena.Json
+{
+ public interface IJsonCollection : ICollection
+ {
+ void Dump ();
+ void Dump (int count);
+ }
+}
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/JsonArray.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/JsonArray.cs
new file mode 100644
index 0000000..f2288bf
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/JsonArray.cs
@@ -0,0 +1,60 @@
+//
+// JsonArray.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+using System;
+using System.Collections.Generic;
+
+namespace Hyena.Json
+{
+ public class JsonArray : List<object>, IJsonCollection
+ {
+ public void Dump ()
+ {
+ Dump (1);
+ }
+
+ public void Dump (int levels)
+ {
+ if (Count == 0) {
+ Console.WriteLine ("[ ]");
+ return;
+ }
+
+ Console.WriteLine ("[");
+ foreach (object item in this) {
+ Console.Write (String.Empty.PadLeft (levels * 2, ' '));
+ if (item is IJsonCollection) {
+ ((IJsonCollection)item).Dump (levels + 1);
+ } else {
+ Console.WriteLine (item);
+ }
+ }
+ Console.WriteLine ("{0}]", String.Empty.PadLeft ((levels - 1) * 2, ' '));
+ }
+ }
+}
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/JsonObject.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/JsonObject.cs
new file mode 100644
index 0000000..98d32c4
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/JsonObject.cs
@@ -0,0 +1,60 @@
+//
+// JsonObject.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+using System;
+using System.Collections.Generic;
+
+namespace Hyena.Json
+{
+ public class JsonObject : Dictionary<string, object>, IJsonCollection
+ {
+ public void Dump ()
+ {
+ Dump (1);
+ }
+
+ public void Dump (int levels)
+ {
+ if (Count == 0) {
+ Console.WriteLine ("{ }");
+ return;
+ }
+
+ Console.WriteLine ("{");
+ foreach (KeyValuePair<string, object> item in this) {
+ Console.Write ("{0}\"{1}\" : ", String.Empty.PadLeft (levels * 2, ' '), item.Key);
+ if (item.Value is IJsonCollection) {
+ ((IJsonCollection)item.Value).Dump (levels + 1);
+ } else {
+ Console.WriteLine (item.Value);
+ }
+ }
+ Console.WriteLine ("{0}}}", String.Empty.PadLeft ((levels - 1) * 2, ' '));
+ }
+ }
+}
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/Tests/DeserializerTests.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/Tests/DeserializerTests.cs
new file mode 100644
index 0000000..e5c9927
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/Tests/DeserializerTests.cs
@@ -0,0 +1,102 @@
+//
+// DeserializerTests.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+#if ENABLE_TESTS
+
+using System;
+using System.Reflection;
+using NUnit.Framework;
+
+using Hyena.Json;
+
+namespace Hyena.Json.Tests
+{
+ [TestFixture]
+ public class DeserializerTests : Hyena.Tests.TestBase
+ {
+ private Deserializer deserializer;
+
+ [TestFixtureSetUp]
+ public void Setup ()
+ {
+ deserializer = new Deserializer ();
+ }
+
+ [Test]
+ public void Literal ()
+ {
+ Assert.AreEqual ("hello", (string)deserializer.SetInput ("\"hello\"").Deserialize ());
+ Assert.AreEqual (-1.76e-3, (double)deserializer.SetInput ("-1.76e-3").Deserialize ());
+ Assert.AreEqual (null, deserializer.SetInput ("null").Deserialize ());
+ Assert.AreEqual (true, (bool)deserializer.SetInput ("true").Deserialize ());
+ Assert.AreEqual (false, (bool)deserializer.SetInput ("false").Deserialize ());
+ }
+
+ [Test]
+ public void Array ()
+ {
+ JsonArray array = (JsonArray)deserializer.SetInput ("[]").Deserialize ();
+ Assert.AreEqual (0, array.Count);
+
+ array = (JsonArray)deserializer.SetInput ("[[]]").Deserialize ();
+ Assert.AreEqual (1, array.Count);
+ Assert.AreEqual (0, ((JsonArray)array[0]).Count);
+
+ array = (JsonArray)deserializer.SetInput ("[[true,[]]]").Deserialize ();
+ Assert.AreEqual (1, array.Count);
+ Assert.AreEqual (2, ((JsonArray)array[0]).Count);
+ Assert.AreEqual (0, ((JsonArray)((JsonArray)array[0])[1]).Count);
+
+ array = (JsonArray)deserializer.SetInput ("[\"a\", 1.0, true]").Deserialize ();
+ Assert.AreEqual (3, array.Count);
+ Assert.AreEqual ("a", (string)array[0]);
+ Assert.AreEqual (1, (double)array[1]);
+ Assert.AreEqual (true, (bool)array[2]);
+ }
+
+ [Test]
+ public void Object ()
+ {
+ JsonObject obj = (JsonObject)deserializer.SetInput ("{}").Deserialize ();
+ Assert.AreEqual (0, obj.Count);
+
+ obj = (JsonObject)deserializer.SetInput ("{\"a\":{}}").Deserialize ();
+ Assert.AreEqual (1, obj.Count);
+ Assert.AreEqual (0, ((JsonObject)obj["a"]).Count);
+
+ obj = (JsonObject)deserializer.SetInput ("{\"a\":[{\"b\":false},\"c\"]}").Deserialize ();
+ Assert.AreEqual (1, obj.Count);
+ JsonArray arr = (JsonArray)obj["a"];
+ Assert.AreEqual (2, arr.Count);
+ Assert.AreEqual (false, ((JsonObject)arr[0])["b"]);
+ Assert.AreEqual ("c", (string)arr[1]);
+ }
+ }
+}
+
+#endif
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/Tests/TokenizerTests.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/Tests/TokenizerTests.cs
new file mode 100644
index 0000000..696fa95
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/Tests/TokenizerTests.cs
@@ -0,0 +1,207 @@
+//
+// TokenizerTests.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+#if ENABLE_TESTS
+
+using System;
+using System.Reflection;
+using NUnit.Framework;
+
+using Hyena.Json;
+
+namespace Hyena.Json.Tests
+{
+ [TestFixture]
+ public class TokenizerTests : Hyena.Tests.TestBase
+ {
+ private Tokenizer tokenizer;
+
+ [TestFixtureSetUp]
+ public void Setup ()
+ {
+ tokenizer = new Tokenizer ();
+ }
+
+ [Test]
+ public void Whitespace ()
+ {
+ AssertTokenStream ("");
+ AssertTokenStream (" ");
+ AssertTokenStream ("\f\n\r\t ");
+ }
+
+ [Test]
+ public void BoolNull ()
+ {
+ // Boolean/null tests
+ AssertTokenStream ("true", Token.Bool (true));
+ AssertTokenStream ("false", Token.Bool (false));
+ AssertTokenStream ("null", Token.Null);
+ }
+
+ [Test]
+ public void NumberInt ()
+ {
+ // Number tests
+ AssertTokenStream ("0", Token.Number (0));
+ AssertTokenStream ("-0", Token.Number (-0));
+
+ AssertTokenStream ("9", Token.Number (9));
+ AssertTokenStream ("-9", Token.Number (-9));
+
+ AssertTokenStream ("14", Token.Number (14));
+ AssertTokenStream ("-14", Token.Number (-14));
+
+ AssertTokenStream ("15309", Token.Number (15309));
+ AssertTokenStream ("-15309", Token.Number (-15309));
+ }
+
+ [Test]
+ public void NumberFloat ()
+ {
+ AssertTokenStream ("0.0", Token.Number (0.0));
+ AssertTokenStream ("-0.0", Token.Number (-0.0));
+
+ AssertTokenStream ("1.9", Token.Number (1.9));
+ AssertTokenStream ("-1.9", Token.Number (-1.9));
+
+ AssertTokenStream ("9.1", Token.Number (9.1));
+ AssertTokenStream ("-9.1", Token.Number (-9.1));
+
+ AssertTokenStream ("15309.0", Token.Number (15309.0));
+ AssertTokenStream ("15309.9", Token.Number (15309.9));
+ AssertTokenStream ("-15309.01", Token.Number (-15309.01));
+ AssertTokenStream ("-15309.9009", Token.Number (-15309.9009));
+ }
+
+ [Test]
+ public void NumberExponent ()
+ {
+ AssertTokenStream ("20.6e3", Token.Number (20.6e3));
+ AssertTokenStream ("20.6e+3", Token.Number (20.6e+3));
+ AssertTokenStream ("20.6e-3", Token.Number (20.6e-3));
+ AssertTokenStream ("-20.6e3", Token.Number (-20.6e3));
+ AssertTokenStream ("-20.6e+3", Token.Number (-20.6e+3));
+ AssertTokenStream ("-20.6e-3", Token.Number (-20.6e-3));
+
+ AssertTokenStream ("1e1", Token.Number (1e1));
+ AssertTokenStream ("1E2", Token.Number (1E2));
+ AssertTokenStream ("1.0e1", Token.Number (1.0e1));
+ AssertTokenStream ("1.0E1", Token.Number (1.0E1));
+ }
+
+ [Test]
+ public void Strings ()
+ {
+ AssertTokenStream (@"""""", Token.String (""));
+ AssertTokenStream (@"""a""", Token.String ("a"));
+ AssertTokenStream (@"""ab""", Token.String ("ab"));
+ AssertTokenStream (@" ""a b"" ", Token.String ("a b"));
+ AssertTokenStream (@"""\""\""""", Token.String ("\"\""));
+ AssertTokenStream (@" ""a \"" \"" b"" ", Token.String ("a \" \" b"));
+ AssertTokenStream (@"""\ubeef""", Token.String ("\ubeef"));
+ AssertTokenStream (@"""\u00a9""", Token.String ("\u00a9"));
+ AssertTokenStream (@"""\u0000\u0001\u0002""", Token.String ("\u0000\u0001\u0002"));
+ AssertTokenStream (@"""1\uabcdef0""", Token.String ("1\uabcdef0"));
+ AssertTokenStream (@"""\b\f\n\r\t""", Token.String ("\b\f\n\r\t"));
+ }
+
+ [Test]
+ public void Container ()
+ {
+ AssertTokenStream ("{}", Token.ObjectStart, Token.ObjectFinish);
+ AssertTokenStream ("[]", Token.ArrayStart, Token.ArrayFinish);
+ AssertTokenStream ("{ }", Token.ObjectStart, Token.ObjectFinish);
+ AssertTokenStream ("[ ]", Token.ArrayStart, Token.ArrayFinish);
+ AssertTokenStream ("[{}]", Token.ArrayStart, Token.ObjectStart, Token.ObjectFinish, Token.ArrayFinish);
+ AssertTokenStream ("[[[ { } ]]]",
+ Token.ArrayStart, Token.ArrayStart, Token.ArrayStart,
+ Token.ObjectStart, Token.ObjectFinish,
+ Token.ArrayFinish, Token.ArrayFinish, Token.ArrayFinish);
+ }
+
+ [Test]
+ public void Array ()
+ {
+ AssertTokenStream ("[1]", Token.ArrayStart, Token.Number (1), Token.ArrayFinish);
+ AssertTokenStream ("[1,0]", Token.ArrayStart, Token.Number (1), Token.Comma, Token.Number (0), Token.ArrayFinish);
+ AssertTokenStream ("[\"a\",true,null]", Token.ArrayStart, Token.String ("a"), Token.Comma,
+ Token.Bool (true), Token.Comma, Token.Null, Token.ArrayFinish);
+ AssertTokenStream ("[0,1,[[2,[4]],5],6]", Token.ArrayStart, Token.Number (0), Token.Comma, Token.Number (1),
+ Token.Comma, Token.ArrayStart, Token.ArrayStart, Token.Number (2), Token.Comma, Token.ArrayStart,
+ Token.Number (4), Token.ArrayFinish, Token.ArrayFinish, Token.Comma, Token.Number (5), Token.ArrayFinish,
+ Token.Comma, Token.Number (6), Token.ArrayFinish);
+ }
+
+ [Test]
+ public void Object ()
+ {
+ AssertTokenStream ("{\"a\":{}}", Token.ObjectStart, Token.String ("a"), Token.Colon, Token.ObjectStart,
+ Token.ObjectFinish, Token.ObjectFinish);
+ AssertTokenStream ("{\"a\":{\"b\":[],\"c\":false}}", Token.ObjectStart, Token.String ("a"),
+ Token.Colon, Token.ObjectStart, Token.String ("b"), Token.Colon, Token.ArrayStart, Token.ArrayFinish,
+ Token.Comma, Token.String ("c"), Token.Colon, Token.Bool (false), Token.ObjectFinish, Token.ObjectFinish);
+ AssertTokenStream ("[{\"a\":{},{}]", Token.ArrayStart, Token.ObjectStart, Token.String ("a"), Token.Colon,
+ Token.ObjectStart, Token.ObjectFinish, Token.Comma, Token.ObjectStart, Token.ObjectFinish, Token.ArrayFinish);
+ }
+
+ private void AssertTokenStream (string input, params Token [] tokens)
+ {
+ int cmp_idx = 0;
+ tokenizer.SetInput (input);
+
+ while (true) {
+ Token token = tokenizer.Scan ();
+ if (token == null) {
+ if (cmp_idx != tokens.Length) {
+ throw new ApplicationException ("Unexpected EOF");
+ }
+ break;
+ }
+
+ Token compare = tokens[cmp_idx++];
+ if (compare.Type != token.Type) {
+ throw new ApplicationException (String.Format ("TokenTypes do not match (exp {0}, got {1}",
+ compare.Type, token.Type));
+ }
+
+ if (compare.Value == null && token.Value == null) {
+ continue;
+ }
+
+ if ((compare.Type == TokenType.Number && (double)compare.Value != (double)token.Value) ||
+ (compare.Type == TokenType.String && (string)compare.Value != (string)token.Value) ||
+ (compare.Type == TokenType.Boolean && (bool)compare.Value != (bool)token.Value)) {
+ throw new ApplicationException ("Token values do not match");
+ }
+ }
+ }
+ }
+}
+
+#endif
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/Token.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/Token.cs
new file mode 100644
index 0000000..cce60c7
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/Token.cs
@@ -0,0 +1,111 @@
+//
+// Token.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+using System;
+
+namespace Hyena.Json
+{
+ internal class Token
+ {
+ public Token (TokenType type) : this (type, null)
+ {
+ }
+
+ public Token (TokenType type, object value)
+ {
+ this.type = type;
+ this.value = value;
+ }
+
+ private TokenType type;
+ public TokenType Type {
+ get { return type; }
+ }
+
+ private object value;
+ public object Value {
+ get { return value; }
+ set { this.value = value; }
+ }
+
+ private int source_line;
+ public int SourceLine {
+ get { return source_line; }
+ internal set { source_line = value; }
+ }
+
+ private int source_column;
+ public int SourceColumn {
+ get { return source_column; }
+ internal set { source_column = value; }
+ }
+
+ internal static Token ObjectStart {
+ get { return new Token (TokenType.ObjectStart); }
+ }
+
+ internal static Token ObjectFinish {
+ get { return new Token (TokenType.ObjectFinish); }
+ }
+
+ internal static Token ArrayStart {
+ get { return new Token (TokenType.ArrayStart); }
+ }
+
+ internal static Token ArrayFinish {
+ get { return new Token (TokenType.ArrayFinish); }
+ }
+
+ internal static Token Null {
+ get { return new Token (TokenType.Null); }
+ }
+
+ internal static Token Comma {
+ get { return new Token (TokenType.Comma); }
+ }
+
+ internal static Token Colon {
+ get { return new Token (TokenType.Colon); }
+ }
+
+ internal static Token Number (double value)
+ {
+ return new Token (TokenType.Number, value);
+ }
+
+ internal static Token String (string value)
+ {
+ return new Token (TokenType.String, value);
+ }
+
+ internal static Token Bool (bool value)
+ {
+ return new Token (TokenType.Boolean, value);
+ }
+ }
+}
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/TokenType.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/TokenType.cs
new file mode 100644
index 0000000..b24a5ca
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/TokenType.cs
@@ -0,0 +1,51 @@
+//
+// TokenType.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+using System;
+
+namespace Hyena.Json
+{
+ [Flags]
+ internal enum TokenType
+ {
+ None = 0 << 0,
+ ObjectStart = 1 << 0,
+ ObjectFinish = 1 << 1,
+ ArrayStart = 1 << 2,
+ ArrayFinish = 1 << 3,
+ Boolean = 1 << 4,
+ String = 1 << 5,
+ Null = 1 << 6,
+ Number = 1 << 7,
+ Comma = 1 << 8,
+ Colon = 1 << 9,
+
+ Literal = String | Number | Boolean | Null,
+ Value = ObjectStart | ArrayStart | Literal
+ }
+}
diff --git a/Tomboy/Addins/WebSyncService/Hyena.Json/Tokenizer.cs b/Tomboy/Addins/WebSyncService/Hyena.Json/Tokenizer.cs
new file mode 100644
index 0000000..f866c0b
--- /dev/null
+++ b/Tomboy/Addins/WebSyncService/Hyena.Json/Tokenizer.cs
@@ -0,0 +1,316 @@
+//
+// Tokenizer.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright (C) 2008 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+using System;
+using System.IO;
+using System.Text;
+
+namespace Hyena.Json
+{
+ internal class Tokenizer
+ {
+ private StreamReader reader;
+ private StringBuilder string_buffer;
+
+ private char peek = ' ';
+ private int current_line = 1;
+ private int current_column = 1;
+ private int token_start_line;
+ private int token_start_column;
+
+ public Tokenizer () { Reset (); }
+ public Tokenizer (string input) { SetInput (input); }
+ public Tokenizer (Stream stream) { SetInput (stream); }
+ public Tokenizer (StreamReader reader) { SetInput (reader); }
+
+ private void Reset ()
+ {
+ peek = ' ';
+ current_line = 1;
+ current_column = 1;
+ token_start_line = 0;
+ token_start_column = 0;
+ }
+
+ public void SetInput (StreamReader reader)
+ {
+ this.reader = reader;
+ Reset ();
+ }
+
+ public void SetInput (Stream stream)
+ {
+ SetInput (new StreamReader (stream));
+ }
+
+ public void SetInput (string input)
+ {
+ SetInput (new MemoryStream (Encoding.UTF8.GetBytes (input)));
+ }
+
+ private void ReadChar ()
+ {
+ peek = (char)reader.Read ();
+ current_column++;
+ }
+
+ private void UnexpectedCharacter (char ch)
+ {
+ throw new ApplicationException (String.Format ("Unexpected character '{0}' at [{1}:{2}]",
+ ch, current_line, current_column - 1));
+ }
+
+ private void InvalidSyntax (string message)
+ {
+ throw new ApplicationException (String.Format ("Invalid syntax: {0} at [{1}:{2}]",
+ message, current_line, current_column));
+ }
+
+ private StringBuilder GetStringBuilder ()
+ {
+ if (string_buffer == null) {
+ string_buffer = new StringBuilder (64);
+ return string_buffer;
+ }
+
+ string_buffer.Remove (0, string_buffer.Length);
+ return string_buffer;
+ }
+
+ private string LexString ()
+ {
+ StringBuilder buffer = GetStringBuilder ();
+ bool read = true;
+
+ while (!reader.EndOfStream) {
+ if (read) {
+ ReadChar ();
+ }
+
+ read = true;
+
+ if (peek == '\\') {
+ ReadChar ();
+ switch (peek) {
+ case 'u':
+ ReadChar ();
+ buffer.Append ((char)LexInt (true, 4));
+ read = false;
+ break;
+ case '"':
+ case '\\':
+ case '/': buffer.Append (peek); break;
+ case 'b': buffer.Append ('\b'); break;
+ case 'f': buffer.Append ('\f'); break;
+ case 'n': buffer.Append ('\n'); break;
+ case 'r': buffer.Append ('\r'); break;
+ case 't': buffer.Append ('\t'); break;
+ default:
+ UnexpectedCharacter (peek);
+ break;
+ }
+ } else if (peek == '"') {
+ ReadChar ();
+ return buffer.ToString ();
+ } else {
+ buffer.Append (peek);
+ }
+ }
+
+ if (peek != '"') {
+ InvalidSyntax ("Unterminated string, expected '\"' termination, got '" + peek + "'");
+ } else if (!read && reader.EndOfStream) {
+ ReadChar ();
+ }
+
+ return buffer.ToString ();
+ }
+
+ private string LexId ()
+ {
+ StringBuilder buffer = GetStringBuilder ();
+
+ do {
+ buffer.Append (peek);
+ ReadChar ();
+ } while (Char.IsLetterOrDigit (peek));
+
+ return buffer.ToString ();
+ }
+
+ private double LexInt ()
+ {
+ return LexInt (false, 0);
+ }
+
+ private double LexInt (bool hex, int maxDigits)
+ {
+ double value = 0.0;
+ int count = 0;
+
+ do {
+ value = (hex ? 16 : 10) * value + (hex
+ ? peek >= 'A' && peek <= 'F'
+ ? 10 + peek - 'A'
+ : (peek >= 'a' && peek <= 'f'
+ ? 10 + peek - 'a'
+ : peek - '0')
+ : peek - '0');
+
+ if (maxDigits > 0 && ++count >= maxDigits) {
+ ReadChar ();
+ return value;
+ }
+
+ ReadChar ();
+ } while (Char.IsDigit (peek) || (hex && ((peek >= 'a' && peek <= 'f') || (peek >= 'A' && peek <= 'F'))));
+
+ return value;
+ }
+
+ private double LexFraction ()
+ {
+ double fraction = 0;
+ double d = 10;
+
+ while (true) {
+ ReadChar ();
+
+ if (!Char.IsDigit (peek)) {
+ break;
+ }
+
+ fraction += (peek - '0') / d;
+ d *= 10;
+ }
+
+ return fraction;
+ }
+
+ private double LexNumber ()
+ {
+ double value = 0.0;
+ bool negate = peek == '-';
+ if (negate) {
+ ReadChar ();
+ }
+
+ if (peek != '0') {
+ value = LexInt ();
+ } else {
+ ReadChar ();
+ }
+
+ if (peek == '.') {
+ value += LexFraction ();
+ }
+
+ if (peek == 'e' || peek == 'E') {
+ ReadChar ();
+ if (peek == '-') {
+ ReadChar ();
+ value /= Math.Pow (10, LexInt ());
+ } else if (peek == '+') {
+ ReadChar ();
+ value *= Math.Pow (10, LexInt ());
+ } else if (Char.IsDigit (peek)) {
+ value *= Math.Pow (10, LexInt ());
+ } else {
+ InvalidSyntax ("Malformed exponent");
+ }
+ }
+
+ if (Char.IsDigit (peek)) {
+ InvalidSyntax ("Numbers starting with 0 must be followed by a . or not " +
+ "followed by a digit (octal syntax not legal)");
+ }
+
+ return negate ? -1.0 * value : value;
+ }
+
+ public Token Scan ()
+ {
+ Token token = InnerScan ();
+ if (token == null) {
+ return null;
+ }
+
+ token.SourceLine = token_start_line;
+ token.SourceColumn = token_start_column - 1;
+ return token;
+ }
+
+ private Token InnerScan ()
+ {
+ for (; ; ReadChar ()) {
+ if (Char.IsWhiteSpace (peek) && peek != '\n') {
+ continue;
+ } else if (peek == '\n') {
+ current_line++;
+ current_column = 0;
+ } else {
+ break;
+ }
+ }
+
+ token_start_column = current_column;
+ token_start_line = current_line;
+
+ switch (peek) {
+ case '{': ReadChar (); return new Token (TokenType.ObjectStart);
+ case '}': ReadChar (); return new Token (TokenType.ObjectFinish);
+ case '[': ReadChar (); return new Token (TokenType.ArrayStart);
+ case ']': ReadChar (); return new Token (TokenType.ArrayFinish);
+ case ',': ReadChar (); return new Token (TokenType.Comma);
+ case ':': ReadChar (); return new Token (TokenType.Colon);
+ case '"': return new Token (TokenType.String, LexString ());
+ default:
+ if (peek == '-' || Char.IsDigit (peek)) {
+ return new Token (TokenType.Number, LexNumber ());
+ } else if (Char.IsLetter (peek)) {
+ string identifier = LexId ();
+ switch (identifier) {
+ case "true": return new Token (TokenType.Boolean, true);
+ case "false": return new Token (TokenType.Boolean, false);
+ case "null": return new Token (TokenType.Null);
+ default:
+ InvalidSyntax ("Invalid identifier '" + identifier + "'");
+ break;
+ }
+ }
+
+ if (peek != Char.MaxValue) {
+ UnexpectedCharacter (peek);
+ }
+ break;
+ }
+
+ return null;
+ }
+ }
+}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
