Streaming JSON

// `Parse` builds the whole tree before a program sees any of it, so a large document costs its
// size several times over in memory. `JsonEventReader` reads the same text as a stream of
// events instead — an object opened, a name, a text value, an array closed — pulling bytes from
// a `Reader` as it needs them and keeping only the token it is on. The program keeps whatever it
// wants and lets the rest go by.
//
// Streaming moves two jobs onto the program:
//
//   - Lifetimes. `Text()` borrows the reader's buffer, which the next `Next()` reuses, so the text
//     of an event must be used or copied before asking for another. The reader owns that buffer,
//     and its destructor gives it back however the function holding it ends.
//   - Errors. `Next()` answers `JsonEvent::Error` when the document goes wrong, and `Failure()`
//     then says why and at which byte of the whole document, however it was split into reads.
//     A source that fails is reported as `JsonParseError::Source`, its own `IoError` kept by
//     `SourceError()`. Events before the mistake were already delivered, so a program must be
//     ready to find out late that its input was bad.
import Allocator::{ Allocator, SystemAllocator };
import Io::{ IoError, IoErrorKind, PrintLine, Reader };
import Json::{ JsonEvent, JsonEventReader, JsonLimits, JsonParseError };
import Text::StringView;

// Stands in for a file or a socket: serves its text at most 8 bytes per read, so tokens arrive
// split across reads, as they would from a real stream.
struct TextSource {
    text: char8[..];
    offset: uint;
}

extend TextSource : Reader {
    func Read(self: &var TextSource, into: var char8[..]) -> uint ! IoError {
        let remaining = self.text.length - self.offset;
        if remaining == 0 {
            fail IoError::Of(IoErrorKind::EndOfStream);
        }
        var count: uint = 8;
        if remaining < count { count = remaining; }
        if into.length < count { count = into.length; }
        for i in 0..count {
            into[i] = self.text[self.offset + i];
        }
        self.offset += count;
        return count;
    }
}

func IsTitle(text: char8[..]) -> bool {
    let title = StringView::FromValidated("title");
    return StringView::FromValidated(text).Equals(title);
}

// Prints the title of every book and counts the books, without ever building a tree.
func Titles(allocator: Allocator, document: char8[..]) -> uint ! (IoError | JsonParseError) {
    let source: Reader = TextSource { text: document, offset: 0 };
    // The reader's buffer lives as long as `reader` does. Whether this function ends at the
    // `return` or at an early `fail`, the destructor frees it on the way out.
    var reader = JsonEventReader::New(allocator, source, JsonLimits::Default())?;
    var books: uint = 0;
    var wantTitle = false;
    var event = reader.Next();
    while event != JsonEvent::End {
        if event == JsonEvent::Error {
            // After an `Error` event `Failure()` always holds the reason; the fallback is
            // only there because the type allows `none`.
            fail reader.Failure() ?? JsonParseError::Unexpected(0);
        }
        // Each book is an object one level inside the top-level array.
        if event == JsonEvent::ObjectStart && reader.Depth() == 2 {
            books += 1;
        }
        // Print the title now: its text is gone once `Next()` is called again.
        if event == JsonEvent::Text && wantTitle {
            PrintLine("  title: {}", reader.Text());
        }
        wantTitle = event == JsonEvent::Name && IsTitle(reader.Text());
        event = reader.Next();
    }
    return books;
}

func Report(allocator: Allocator, document: char8[..]) {
    PrintLine("{}", document);
    match Titles(allocator, document) {
        .Success(books) => PrintLine("  {} books", books),
        .Failure(error) => match error {
            reason: JsonParseError =>
                PrintLine("  refused at byte {}: {}", reason.Offset(), reason),
            _: IoError => PrintLine("  could not start reading")
        }
    }
}

func Main() -> int {
    var system = SystemAllocator();
    let allocator: Allocator = system;
    let catalog = "[{\"title\": \"Dune\", \"year\": 1965}, {\"year\": 1815, \"title\": \"Emma\"}]";
    Report(allocator, catalog);

    // Two broken documents. In both, titles were printed before the mistake was found: a
    // stream cannot know the end is wrong until it gets there.
    Report(allocator, "[{\"title\": \"Dune\"}, {\"title\": \"Emma\"]");
    Report(allocator, "[{\"title\": \"Dune\"}, {\"title\":");
    return 0;
}