Skip to content

Commit

Permalink
Indexing & other features
Browse files Browse the repository at this point in the history 
- Add dot notation (a.b), index notation (a[b]), and function calls.
- Can access date fields using dot notation (date.year, date.month,
etc).
- Can properly render objects, links, and arrays.
- Links now work properly in frontmatter.
- Add tests & improve stability.
- Better rendering for errors.
- Field negation AND source negation
  • Loading branch information
Michael Brenan committed Mar 6, 2021
1 parent 374a666 commit 3fafd0d
Show file tree
Hide file tree
Showing 9 changed files with 242 additions and 131 deletions.
32 changes: 23 additions & 9 deletions src/engine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import { DateTime, Duration } from 'luxon';
import { TFile } from 'obsidian';
import { EXPRESSION } from './parse';
import { Context, BINARY_OPS } from './eval';
import { create } from 'domain';

/** The result of executing a query over an index. */
export interface QueryResult {
Expand Down Expand Up @@ -84,6 +83,11 @@ export function parseFrontmatter(value: any): LiteralField {
} else if (typeof value === 'object') {
if (Array.isArray(value)) {
let object = (value as Array<any>);
// Special case for link syntax, which shows up as double-nested arrays.
if (object.length == 1 && Array.isArray(object[0]) && (object[0].length == 1) && typeof object[0][0] === 'string') {
return Fields.link(object[0][0]);
}

let result = [];
for (let child of object) {
result.push(parseFrontmatter(child));
Expand Down Expand Up @@ -135,31 +139,41 @@ export function createContext(file: string, index: FullIndex): Context {
// Create a context which uses the cache to look up link info.
let context = new Context((file) => {
let meta = index.metadataCache.getCache(file);
if (meta && meta.frontmatter) return parseFrontmatter(meta.frontmatter) as LiteralFieldRepr<'object'>;
if (!meta) {
file += ".md";
meta = index.metadataCache.getCache(file);
}

// TODO: Hacky, change this later.
if (meta && meta.frontmatter) return createContext(file, index).namespace;
else return Fields.NULL;
}, frontmatterData);

// TODO: Make this a real object instead of a fake one.
context.set("file.path", Fields.literal('string', file));
context.set("file.name", Fields.literal('string', getFileName(file)));
// Fill out per-file metadata.
let fileMeta = new Map<string, LiteralField>();
fileMeta.set("path", Fields.literal('string', file));
fileMeta.set("name", Fields.literal('string', getFileName(file)));
fileMeta.set("link", Fields.link(file));

// If the file has a date name, add it as the 'day' field.
let dateMatch = /(\d{4})-(\d{2})-(\d{2})/.exec(getFileName(file));
if (dateMatch) {
let year = Number.parseInt(dateMatch[1]);
let month = Number.parseInt(dateMatch[2]);
let day = Number.parseInt(dateMatch[3]);
context.set("file.day", Fields.literal('date', DateTime.fromObject({ year, month, day })))
fileMeta.set("day", Fields.literal('date', DateTime.fromObject({ year, month, day })))
}

// Populate file metadata.
let afile = index.vault.getAbstractFileByPath(file);
if (afile && afile instanceof TFile) {
context.set('file.ctime', Fields.literal('date', DateTime.fromMillis(afile.stat.ctime)));
context.set('file.mtime', Fields.literal('date', DateTime.fromMillis(afile.stat.mtime)));
context.set('file.size', Fields.literal('number', afile.stat.size));
fileMeta.set('ctime', Fields.literal('date', DateTime.fromMillis(afile.stat.ctime)));
fileMeta.set('mtime', Fields.literal('date', DateTime.fromMillis(afile.stat.mtime)));
fileMeta.set('size', Fields.literal('number', afile.stat.size));
}

context.set("file", Fields.object(fileMeta));

return context;
}

Expand Down
126 changes: 67 additions & 59 deletions src/eval.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export type LinkResolverImpl = (file: string) => LiteralFieldRepr<'object'> | Li
/** The context in which expressions are evaluated in. */
export class Context {
/** Direct variable fields in the context. */
private namespace: LiteralFieldRepr<'object'>;
public namespace: LiteralFieldRepr<'object'>;
/** Registry of binary operation handlers. */
public readonly binaryOps: BinaryOpHandler;
/** Registry of function handlers. */
Expand All @@ -33,33 +33,9 @@ export class Context {
return this;
}

/** Attempts to resolve a field name relative to the given field. */
public get(name: string, root: LiteralField = this.namespace): LiteralField {
let parts = name.split(".");

let current: LiteralField = root;
for (let index = 0; index < parts.length; index++) {
let next: LiteralField | undefined = undefined;

switch (current.valueType) {
case "object":
next = current.value.get(parts[index]);
break;
case "link":
let data = this.linkResolver(current.value);
if (data.valueType == 'null') return Fields.NULL;
next = data.value.get(parts[index]);
break;
default:
// Trying to subindex into a non-container type.
return Fields.NULL;
}

if (next == undefined) return Fields.NULL;
current = next;
}

return current;
/** Attempts to resolve a variable name in the context. */
public get(name: string): LiteralField {
return this.namespace.value.get(name) ?? Fields.NULL;
}

/** Evaluate a field in this context, returning the final resolved value. */
Expand All @@ -85,10 +61,65 @@ export class Context {
args.push(resolved);
}

let func = this.functions.get(field.func);
if (!func) return `Function ${field.func} does not exist.`;

return func(args, this);
// TODO: Add later support for lambdas as an additional thing you can call.
switch (field.func.type) {
case "variable":
let func = this.functions.get(field.func.name);
if (!func) return `Function ${field.func} does not exist.`;
return func(args, this);
default:
return `Cannot call field '${field.func}' as a function`;
}
case "index":
let obj = this.evaluate(field.object);
if (typeof obj === 'string') return obj;
let index = this.evaluate(field.index);
if (typeof index === 'string') return index;

switch (obj.valueType) {
case "object":
if (index.valueType != 'string') return "can only index into objects with strings (a.b or a[\"b\"])";
return obj.value.get(index.value) ?? Fields.NULL;
case "link":
if (index.valueType != 'string') return "can only index into links with strings (a.b or a[\"b\"])";
let linkValue = this.linkResolver(obj.value);
if (linkValue.valueType == 'null') return Fields.NULL;
return linkValue.value.get(index.value) ?? Fields.NULL;
case "array":
if (index.valueType != 'number') return "array indexing requires a numeric index (array[index])";
if (index.value >= obj.value.length || index.value < 0) return Fields.NULL;
return obj.value[index.value];
case "string":
if (index.valueType != 'number') return "string indexing requires a numeric index (string[index])";
if (index.value >= obj.value.length || index.value < 0) return Fields.NULL;
return Fields.string(obj.value[index.value]);
case "date":
if (index.valueType != 'string') return "date indexing requires a string representing the unit";
switch (index.value) {
case "year": return Fields.number(obj.value.year);
case "month": return Fields.number(obj.value.month);
case "day": return Fields.number(obj.value.day);
case "hour": return Fields.number(obj.value.hour);
case "minute": return Fields.number(obj.value.minute);
case "second": return Fields.number(obj.value.second);
case "millisecond": return Fields.number(obj.value.millisecond);
default: return Fields.NULL;
}
case "duration":
if (index.valueType != 'string') return "duration indexing requires a string representing the unit";
switch (index.value) {
case "year": case "years": return Fields.number(obj.value.years);
case "month": case "months": return Fields.number(obj.value.months);
case "day": case "days": return Fields.number(obj.value.days);
case "hour": case "hours": return Fields.number(obj.value.hours);
case "minute": case "minutes": return Fields.number(obj.value.minutes);
case "second": case "seconds": return Fields.number(obj.value.seconds);
case "millisecond": case "milliseconds": return Fields.number(obj.value.milliseconds);
default: return Fields.NULL;
}
default:
return Fields.NULL;
}
}
}
}
Expand Down Expand Up @@ -226,8 +257,8 @@ export const BINARY_OPS = BinaryOpHandler.create()
.add('-', 'date', 'duration', (a, b) => Fields.literal('date', a.value.minus(b.value)))
// Link operations.
.addComparison('link', {
equals: (a, b) => Fields.bool(a.value == b.value),
le: (a, b) => Fields.bool(a.value < b.value)
equals: (a, b) => Fields.bool(a.value.replace(".md", "") == b.value.replace(".md", "")),
le: (a, b) => Fields.bool(a.value.replace(".md", "") < b.value.replace(".md", ""))
})
// Array operations.
.add('+', 'array', 'array', (a, b) => Fields.array([].concat(a.value).concat(b.value)))
Expand Down Expand Up @@ -278,6 +309,7 @@ export const FUNCTIONS = new Map<string, FunctionImpl>()
if (args.length == 0 || args.length > 1) return "length() requires exactly 1 argument";
let value = args[0];

// TODO: Add links to this.
switch (value.valueType) {
case "array": return Fields.number(value.value.length);
case "object": return Fields.number(value.value.size);
Expand All @@ -286,28 +318,4 @@ export const FUNCTIONS = new Map<string, FunctionImpl>()
}
})
.set("list", (args, context) => Fields.array(args))
.set("array", (args, context) => Fields.array(args))
.set("get", (args, context) => {
if (args.length != 2) return "get() requires exactly 2 arguments";
let object = args[0];
let index = args[1];

switch (object.valueType) {
case "object":
if (index.valueType != 'string') return "get(object, index) requires a string index";
return context.get(index.value, object);
case "link":
if (index.valueType != 'string') return "get(link, index) requires a string index";
return context.get(index.value, object);
case "array":
if (index.valueType != 'number') return "get(array, index) requires a numeric index";
if (index.value >= object.value.length || index.value < 0) return Fields.NULL;
return object.value[index.value];
case "string":
if (index.valueType != 'number') return "get(string, index) requires a numeric index";
if (index.value >= object.value.length || index.value < 0) return Fields.NULL;
return Fields.string(object.value[index.value]);
}

return "get() can only be used on an object, link, array, or string";
});
.set("array", (args, context) => Fields.array(args));
11 changes: 5 additions & 6 deletions src/legacy-parse.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export const QUERY_LANGUAGE = P.createLanguage<QueryLanguageTypes>({
(field, _1, _2, _3, ident) => Fields.named(ident, field)),
namedField: q => P.alt<NamedField>(
q.explicitNamedField,
EXPRESSION.variableField.map(field => Fields.named(field.name, field))
EXPRESSION.identifierDot.map(ident => Fields.named(ident, Fields.indexVariable(ident)))
),
sortField: q => P.seqMap(P.optWhitespace,
EXPRESSION.field, P.optWhitespace, P.regexp(/ASCENDING|DESCENDING|ASC|DESC/i).atMost(1),
Expand Down Expand Up @@ -116,10 +116,9 @@ export const QUERY_LANGUAGE = P.createLanguage<QueryLanguageTypes>({
* if the parse failed.
*/
export function parseQuery(text: string): Query | string {
let result = QUERY_LANGUAGE.query.parse(text);
if (result.status == true) {
return result.value;
} else {
return `Failed to parse query (line ${result.index.line}, column ${result.index.column}): expected ${result.expected}`;
try {
return QUERY_LANGUAGE.query.tryParse(text);
} catch (error) {
return "" + error;
}
}
10 changes: 2 additions & 8 deletions src/main.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { MarkdownRenderChild, Plugin, Workspace, Vault, MarkdownPostProcessorContext, PluginSettingTab, App, Setting } from 'obsidian';
import { createAnchor, prettifyYamlKey, renderErrorPre, renderList, renderMinimalDate, renderTable } from './render';
import { createAnchor, prettifyYamlKey, renderErrorPre, renderField, renderList, renderMinimalDate, renderTable } from './render';
import { FullIndex, TaskCache } from './index';
import * as Tasks from './tasks';
import { Query } from './query';
Expand Down Expand Up @@ -238,13 +238,7 @@ class DataviewTableRenderer extends MarkdownRenderChild {
[createAnchor(filename, row.file.replace(".md", ""), true)];

for (let elem of row.data) {
if (elem.valueType == 'date') {
result.push(renderMinimalDate(elem.value));
} else if (elem.valueType == 'null') {
result.push(this.settings.renderNullAs);
} else {
result.push("" + elem.value);
}
result.push(renderField(elem, this.settings.renderNullAs));
}

return result;
Expand Down
49 changes: 42 additions & 7 deletions src/parse.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,18 @@ export function chainOpt<T>(base: P.Parser<T>, ...funcs: ((r: T) => P.Parser<T>)
// Expression Parsing //
////////////////////////

type PostfixFragment =
{ 'type': 'dot'; field: Field; }
| { 'type': 'index'; field: Field; }
| { 'type': 'function'; fields: Field[]; };

interface ExpressionLanguage {
number: number;
string: string;
bool: boolean;
tag: string;
identifier: string;
identifierDot: string;
link: string;
rootDate: DateTime;
date: DateTime;
Expand Down Expand Up @@ -104,10 +110,16 @@ interface ExpressionLanguage {
dateField: Field;
durationField: Field;
linkField: Field;
functionField: Field;
negatedField: Field;
atomField: Field;
indexField: Field;

// Postfix parsers for function calls & the like.
dotPostfix: PostfixFragment;
indexPostfix: PostfixFragment;
functionPostfix: PostfixFragment;

// Binary op parsers.
binaryMulDivField: Field;
binaryPlusMinusField: Field;
binaryCompareField: Field;
Expand Down Expand Up @@ -135,7 +147,10 @@ export const EXPRESSION = P.createLanguage<ExpressionLanguage>({
tag: q => P.regexp(/#[\p{Letter}\w/-]+/u).desc("tag ('#hello/stuff')"),

// A variable identifier, which is alphanumeric and must start with a letter.
identifier: q => P.regexp(/[\p{Letter}][\.\p{Letter}\w_-]*/u).desc("variable identifier"),
identifier: q => P.regexp(/[\p{Letter}][\p{Letter}\w_-]*/u).desc("variable identifier"),

// A variable identifier, which is alphanumeric and must start with a letter. Can include dots.
identifierDot: q => P.regexp(/[\p{Letter}][\p{Letter}\.\w_-]*/u).desc("variable identifier"),

// An Obsidian link of the form [[<link>]].
link: q => P.regexp(/\[\[([\p{Letter}\w./-]+)\]\]/u, 1).desc("file link"),
Expand Down Expand Up @@ -207,15 +222,35 @@ export const EXPRESSION = P.createLanguage<ExpressionLanguage>({
durationField: q => P.seqMap(P.string("dur("), P.optWhitespace, q.duration, P.optWhitespace, P.string(")"),
(prefix, _1, dur, _2, postfix) => Fields.literal('duration', dur))
.desc("duration"),
functionField: q => P.seqMap(q.identifier, P.optWhitespace, P.string("("), q.field.sepBy(P.string(",").trim(P.optWhitespace)), P.optWhitespace, P.string(")"),
(name, _1, _2, fields, _3, _4) => Fields.func(name, fields)),
linkField: q => q.link.map(f => Fields.link(f)),
negatedField: q => P.seqMap(P.string("!"), q.atomField, (_, field) => Fields.negate(field)).desc("negated field"),
atomField: q => P.alt(q.negatedField, q.parensField, q.boolField, q.numberField, q.stringField, q.linkField, q.dateField, q.durationField, q.functionField, q.variableField),
atomField: q => P.alt(q.negatedField, q.parensField, q.boolField, q.numberField, q.stringField, q.linkField, q.dateField, q.durationField, q.variableField),
indexField: q => P.seqMap(q.atomField, P.alt(q.dotPostfix, q.indexPostfix, q.functionPostfix).many(), (obj, postfixes) => {
let result = obj;
for (let post of postfixes) {
switch (post.type) {
case "dot":
case "index":
result = Fields.index(result, post.field);
break;
case "function":
result = Fields.func(result, post.fields);
break;
}
}

return result;
}),
negatedField: q => P.seqMap(P.string("!"), q.indexField, (_, field) => Fields.negate(field)).desc("negated field"),
parensField: q => P.seqMap(P.string("("), P.optWhitespace, q.field, P.optWhitespace, P.string(")"), (_1, _2, field, _3, _4) => field),

dotPostfix: q => P.seqMap(P.string("."), q.variableField, (_, field) => { return { type: 'dot', field: Fields.string(field.name) } }),
indexPostfix: q => P.seqMap(P.string("["), P.optWhitespace, q.field, P.optWhitespace, P.string("]"),
(_, _2, field, _3, _4) => { return { type: 'index', field }}),
functionPostfix: q => P.seqMap(P.string("("), P.optWhitespace, q.field.sepBy(P.string(",").trim(P.optWhitespace)), P.optWhitespace, P.string(")"),
(_, _1, fields, _2, _3) => { return { type: 'function', fields }}),

// The precedence hierarchy of operators - multiply/divide, add/subtract, compare, and then boolean operations.
binaryMulDivField: q => createBinaryParser(q.atomField, q.binaryMulDiv, Fields.binaryOp),
binaryMulDivField: q => createBinaryParser(q.indexField, q.binaryMulDiv, Fields.binaryOp),
binaryPlusMinusField: q => createBinaryParser(q.binaryMulDivField, q.binaryPlusMinus, Fields.binaryOp),
binaryCompareField: q => createBinaryParser(q.binaryPlusMinusField, q.binaryCompareOp, Fields.binaryOp),
binaryBooleanField: q => createBinaryParser(q.binaryCompareField, q.binaryBooleanOp, Fields.binaryOp),
Expand Down
Loading

0 comments on commit 3fafd0d

Please sign in to comment.