2025-04-01 10:38:02 +09:00

630 lines
23 KiB
TypeScript

import { String } from '@grafana/lezer-logql';
import {
getHighlighterExpressionsFromQuery,
getLokiQueryType,
isLogsQuery,
isQueryWithLabelFormat,
isQueryWithParser,
isQueryWithError,
parseToNodeNamesArray,
getParserFromQuery,
obfuscate,
requestSupportsSplitting,
isQueryWithRangeVariable,
isQueryPipelineErrorFiltering,
getLogQueryFromMetricsQuery,
getNormalizedLokiQuery,
getNodePositionsFromQuery,
getLogQueryFromMetricsQueryAtPosition,
interpolateShardingSelector,
requestSupportsSharding,
} from './queryUtils';
import { LokiQuery, LokiQueryDirection, LokiQueryType } from './types';
describe('getHighlighterExpressionsFromQuery', () => {
it('returns no expressions for empty query', () => {
expect(getHighlighterExpressionsFromQuery('')).toEqual([]);
});
it('returns no expression for query with empty filter ', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= ``')).toEqual([]);
});
it('returns no expression for query with empty filter and parser', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= `` | json count="counter" | __error__=``')).toEqual([]);
});
it('returns no expression for query with empty filter and chained filter', () => {
expect(
getHighlighterExpressionsFromQuery('{foo="bar"} |= `` |= `highlight` | json count="counter" | __error__=``')
).toEqual(['highlight']);
});
it('returns no expression for query with empty filter, chained and regex filter', () => {
expect(
getHighlighterExpressionsFromQuery(
'{foo="bar"} |= `` |= `highlight` |~ `high.ight` | json count="counter" | __error__=``'
)
).toEqual(['highlight', 'high.ight']);
});
it('returns no expression for query with empty filter, chained and regex quotes filter', () => {
expect(
getHighlighterExpressionsFromQuery(
'{foo="bar"} |= `` |= `highlight` |~ "highlight\\\\d" | json count="counter" | __error__=``'
)
).toEqual(['highlight', 'highlight\\d']);
});
it('returns an expression for query with filter using quotes', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x"')).toEqual(['x']);
});
it('returns an expression for query with filter using backticks', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= `x`')).toEqual(['x']);
});
it('returns expressions for query with filter chain', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ "y"')).toEqual(['x', 'y']);
});
it('returns expressions for query with filter chain using both backticks and quotes', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ `y`')).toEqual(['x', 'y']);
});
it('returns expression for query with log parser', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" | logfmt')).toEqual(['x']);
});
it('returns expressions for query with filter chain followed by log parser', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ "y" | logfmt')).toEqual(['x', 'y']);
});
it('returns drops expressions for query with negative filter chain using quotes', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" != "y"')).toEqual(['x']);
});
it('returns expressions for query with filter chain using backticks', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= `x` |~ `y`')).toEqual(['x', 'y']);
});
it('returns expressions for query with filter chain using quotes and backticks', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ `y`')).toEqual(['x', 'y']);
});
it('returns null if filter term is not wrapped in double quotes', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= x')).toEqual([]);
});
it('escapes filter term if regex filter operator is not used', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x[yz].w"')).toEqual(['x\\[yz\\]\\.w']);
});
it('does not escape filter term if regex filter operator is used', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |~ "x[yz].w" |~ "z.+"')).toEqual(['x[yz].w', 'z.+']);
});
it('removes extra backslash escaping if regex filter operator and quotes are used', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |~ "\\\\w+"')).toEqual(['\\w+']);
});
it('does not remove backslash escaping if regex filter operator and backticks are used', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |~ `\\w+`')).toEqual(['\\w+']);
});
it.each`
input | expected
${'`"test"`'} | ${'"test"'}
${'"`test`"'} | ${'`test`'}
${'`"test"a`'} | ${'"test"a'}
`('should correctly identify the type of quote used in the term', ({ input, expected }) => {
expect(getHighlighterExpressionsFromQuery(`{foo="bar"} |= ${input}`)).toEqual([expected]);
});
it.each(['|=', '|~'])('returns multiple expressions when using or statements', (op: string) => {
expect(getHighlighterExpressionsFromQuery(`{app="frontend"} ${op} "line" or "text"`)).toEqual(['line', 'text']);
});
it.each(['|=', '|~'])('returns multiple expressions when using or statements and ip filters', (op: string) => {
expect(getHighlighterExpressionsFromQuery(`{app="frontend"} ${op} "line" or ip("10.0.0.1")`)).toEqual(['line']);
});
});
describe('getNormalizedLokiQuery', () => {
it('removes deprecated instant property', () => {
const input: LokiQuery = { refId: 'A', expr: 'test1', instant: true };
const output = getNormalizedLokiQuery(input);
expect(output).toStrictEqual({ refId: 'A', expr: 'test1', queryType: LokiQueryType.Range });
});
it('removes deprecated range property', () => {
const input: LokiQuery = { refId: 'A', expr: 'test1', range: true };
const output = getNormalizedLokiQuery(input);
expect(output).toStrictEqual({ refId: 'A', expr: 'test1', queryType: LokiQueryType.Range });
});
it('removes deprecated range and instant properties if query with queryType', () => {
const input: LokiQuery = { refId: 'A', expr: 'test1', range: true, instant: false, queryType: LokiQueryType.Range };
const output = getNormalizedLokiQuery(input);
expect(output).toStrictEqual({ refId: 'A', expr: 'test1', queryType: LokiQueryType.Range });
});
});
describe('getLokiQueryType', () => {
function expectCorrectQueryType(inputProps: Object, outputQueryType: LokiQueryType) {
const input: LokiQuery = { refId: 'A', expr: 'test1', ...inputProps };
const output = getLokiQueryType(input);
expect(output).toStrictEqual(outputQueryType);
}
it('handles no props case', () => {
expectCorrectQueryType({}, LokiQueryType.Range);
});
it('handles old-style instant case', () => {
expectCorrectQueryType({ instant: true, range: false }, LokiQueryType.Instant);
});
it('handles old-style range case', () => {
expectCorrectQueryType({ instant: false, range: true }, LokiQueryType.Range);
});
it('handles new+old style instant', () => {
expectCorrectQueryType({ instant: true, range: false, queryType: LokiQueryType.Range }, LokiQueryType.Range);
});
it('handles new+old style range', () => {
expectCorrectQueryType({ instant: false, range: true, queryType: LokiQueryType.Instant }, LokiQueryType.Instant);
});
it('handles new<>old conflict (new wins), range', () => {
expectCorrectQueryType({ instant: false, range: true, queryType: LokiQueryType.Range }, LokiQueryType.Range);
});
it('handles new<>old conflict (new wins), instant', () => {
expectCorrectQueryType({ instant: true, range: false, queryType: LokiQueryType.Instant }, LokiQueryType.Instant);
});
it('handles invalid new, range', () => {
expectCorrectQueryType({ queryType: 'invalid' }, LokiQueryType.Range);
});
it('handles invalid new, when old-range exists, use old', () => {
expectCorrectQueryType({ instant: false, range: true, queryType: 'invalid' }, LokiQueryType.Range);
});
it('handles invalid new, when old-instant exists, use old', () => {
expectCorrectQueryType({ instant: true, range: false, queryType: 'invalid' }, LokiQueryType.Instant);
});
});
describe('isQueryWithError', () => {
it('returns false if invalid query', () => {
expect(isQueryWithError('{job="grafana')).toBe(true);
});
it('returns true if valid query', () => {
expect(isQueryWithError('{job="grafana"}')).toBe(false);
});
});
describe('parseToNodeNamesArray', () => {
it('returns on empty query', () => {
expect(parseToNodeNamesArray('{}')).toEqual(['LogQL', 'Expr', 'LogExpr', 'Selector', '⚠']);
});
it('returns on invalid query', () => {
expect(parseToNodeNamesArray('{job="grafana"')).toEqual([
'LogQL',
'Expr',
'LogExpr',
'Selector',
'Matchers',
'Matcher',
'Identifier',
'Eq',
'String',
'⚠',
]);
});
it('returns on valid query', () => {
expect(parseToNodeNamesArray('{job="grafana"}')).toEqual([
'LogQL',
'Expr',
'LogExpr',
'Selector',
'Matchers',
'Matcher',
'Identifier',
'Eq',
'String',
]);
});
});
describe('obfuscate', () => {
it('obfuscates on invalid query', () => {
expect(obfuscate('{job="grafana"')).toEqual('{Identifier=String');
});
it('obfuscates on valid query', () => {
expect(
obfuscate('sum(sum_over_time({test="test"} |= `` | logfmt | __error__=`` | unwrap test | __error__=`` [10m]))')
).toEqual(
'sum(sum_over_time({Identifier=String} |= String | logfmt | __error__=String | unwrap Identifier | __error__=String [10m]))'
);
});
it('obfuscates on arithmetic operation', () => {
expect(obfuscate('2 + 3')).toEqual('Number + Number');
});
it('obfuscates a comment', () => {
expect(obfuscate('{job="grafana"} # test comment')).toEqual('{Identifier=String} LineComment');
});
it('does not obfuscate interval variables', () => {
expect(
obfuscate(
'sum(quantile_over_time(0.5, {label="$var"} | logfmt | __error__=`` | unwrap latency | __error__=`` [$__interval]))'
)
).toEqual(
'sum(quantile_over_time(Number, {Identifier=String} | logfmt | __error__=String | unwrap Identifier | __error__=String [$__interval]))'
);
});
});
describe('isLogsQuery', () => {
it('returns false if metrics query', () => {
expect(isLogsQuery('rate({job="grafana"}[5m])')).toBe(false);
});
it('returns true if valid query', () => {
expect(isLogsQuery('{job="grafana"}')).toBe(true);
});
});
describe('isQueryWithParser', () => {
it('returns false if query without parser', () => {
expect(isQueryWithParser('rate({job="grafana" |= "error" }[5m])')).toEqual({
parserCount: 0,
queryWithParser: false,
});
});
it('returns true if log query with parser', () => {
expect(isQueryWithParser('{job="grafana"} | json')).toEqual({ parserCount: 1, queryWithParser: true });
});
it('returns true if metric query with parser', () => {
expect(isQueryWithParser('rate({job="grafana"} | json [5m])')).toEqual({ parserCount: 1, queryWithParser: true });
});
it('returns true if query with json parser with expressions', () => {
expect(isQueryWithParser('rate({job="grafana"} | json foo="bar", bar="baz" [5m])')).toEqual({
parserCount: 1,
queryWithParser: true,
});
});
});
describe('isQueryWithLabelFormat', () => {
it('returns true if log query with label format', () => {
expect(isQueryWithLabelFormat('{job="grafana"} | label_format level=lvl')).toBe(true);
});
it('returns true if metrics query with label format', () => {
expect(isQueryWithLabelFormat('rate({job="grafana"} | label_format level=lvl [5m])')).toBe(true);
});
it('returns false if log query without label format', () => {
expect(isQueryWithLabelFormat('{job="grafana"} | json')).toBe(false);
});
it('returns false if metrics query without label format', () => {
expect(isQueryWithLabelFormat('rate({job="grafana"} [5m])')).toBe(false);
});
});
describe('isQueryWithRangeVariableDuration', () => {
it('identifies queries using $__range variable', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"}[$__range])')).toBe(true);
});
it('identifies queries using $__range_s variable', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"}[$__range_s])')).toBe(true);
});
it('identifies queries using $__range_ms variable', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"}[$__range_ms])')).toBe(true);
});
it('does not return false positives', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"} | logfmt | value="$__range" [5m])')).toBe(false);
expect(isQueryWithRangeVariable('rate({job="grafana"} | logfmt | value="[$__range]" [5m])')).toBe(false);
expect(isQueryWithRangeVariable('rate({job="grafana"} [$range])')).toBe(false);
expect(isQueryWithRangeVariable('rate({job="grafana"} [$_range])')).toBe(false);
expect(isQueryWithRangeVariable('rate({job="grafana"} [$_range_ms])')).toBe(false);
});
});
describe('getParserFromQuery', () => {
it('returns no parser', () => {
expect(getParserFromQuery('{job="grafana"}')).toBeUndefined();
});
it.each(['json', 'logfmt', 'pattern', 'regexp', 'unpack'])('detects %s parser', (parser: string) => {
expect(getParserFromQuery(`{job="grafana"} | ${parser}`)).toBe(parser);
expect(getParserFromQuery(`sum(count_over_time({place="luna"} | ${parser} | unwrap counter )) by (place)`)).toBe(
parser
);
});
it('supports json parser with arguments', () => {
// Redundant, but gives us a baseline
expect(getParserFromQuery('{job="grafana"} | json')).toBe('json');
expect(getParserFromQuery('{job="grafana"} | json field="otherField"')).toBe('json');
expect(getParserFromQuery('{job="grafana"} | json field="otherField", label="field2"')).toBe('json');
});
it('supports logfmt parser with arguments and flags', () => {
// Redundant, but gives us a baseline
expect(getParserFromQuery('{job="grafana"} | logfmt')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt --strict')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt --strict --keep-empty')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt field="otherField"')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt field="otherField", label')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt --strict field="otherField"')).toBe('logfmt');
expect(
getParserFromQuery('{job="grafana"} | logfmt --strict --keep-empty field="otherField", label="field2"')
).toBe('logfmt');
});
});
describe('requestSupportsSplitting', () => {
it('hidden requests are not partitioned', () => {
const requests: LokiQuery[] = [
{
expr: '{a="b"}',
refId: 'A',
hide: true,
},
];
expect(requestSupportsSplitting(requests)).toBe(false);
});
it('special requests are not partitioned', () => {
const requests: LokiQuery[] = [
{
expr: '{a="b"}',
refId: 'do-not-chunk',
},
];
expect(requestSupportsSplitting(requests)).toBe(false);
});
it('empty requests are not partitioned', () => {
const requests: LokiQuery[] = [
{
expr: '',
refId: 'A',
},
];
expect(requestSupportsSplitting(requests)).toBe(false);
});
it('all other requests are partitioned', () => {
const requests: LokiQuery[] = [
{
expr: '{a="b"}',
refId: 'A',
},
{
expr: 'count_over_time({a="b"}[1h])',
refId: 'B',
},
];
expect(requestSupportsSplitting(requests)).toBe(true);
});
});
describe('isQueryPipelineErrorFiltering', () => {
it('identifies pipeline error filters', () => {
expect(isQueryPipelineErrorFiltering('{job="grafana"} | logfmt | __error__=""')).toBe(true);
expect(isQueryPipelineErrorFiltering('{job="grafana"} | logfmt | error=""')).toBe(false);
});
});
describe('getLogQueryFromMetricsQuery', () => {
it('returns the log query from a metric query', () => {
expect(getLogQueryFromMetricsQuery('count_over_time({job="grafana"} | logfmt | label="value" [1m])')).toBe(
'{job="grafana"} | logfmt | label="value"'
);
expect(getLogQueryFromMetricsQuery('count_over_time({job="grafana"} [1m])')).toBe('{job="grafana"}');
expect(
getLogQueryFromMetricsQuery(
'sum(quantile_over_time(0.5, {label="$var"} | logfmt | __error__=`` | unwrap latency | __error__=`` [$__interval]))'
)
).toBe('{label="$var"} | logfmt | __error__=``');
});
it('does not return a query when there is no log query', () => {
expect(getLogQueryFromMetricsQuery('1+1')).toBe('');
expect(getLogQueryFromMetricsQuery('count_over_time([1s])')).toBe('');
});
});
describe('getLogQueryFromMetricsQueryAtPosition', () => {
it('works like getLogQueryFromMetricsQuery for simple queries', () => {
expect(
getLogQueryFromMetricsQueryAtPosition('count_over_time({job="grafana"} | logfmt | label="value" [1m])', 57)
).toBe('{job="grafana"} | logfmt | label="value"');
expect(getLogQueryFromMetricsQueryAtPosition('count_over_time({job="grafana"} [1m])', 37)).toBe('{job="grafana"}');
expect(
getLogQueryFromMetricsQueryAtPosition(
'sum(quantile_over_time(0.5, {label="$var"} | logfmt | __error__=`` | unwrap latency | __error__=`` [$__interval]))',
45
)
).toBe('{label="$var"} | logfmt | __error__=``');
});
it.each([
[
'count_over_time({place="moon"} | json test="test" [1m]) + avg_over_time({place="luna"} | logfmt test="test" [1m])',
'{place="moon"} | json test="test"',
49,
],
[
'count_over_time({place="moon"} | json test="test" [1m]) + avg_over_time({place="luna"} | logfmt test="test" [1m])',
'{place="luna"} | logfmt test="test"',
107,
],
])('gets the right query for complex queries', (metric: string, log: string, position: number) => {
expect(getLogQueryFromMetricsQueryAtPosition(metric, position)).toBe(log);
});
});
describe('getNodePositionsFromQuery', () => {
it('returns the right amount of positions without type', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
expect(getNodePositionsFromQuery('{job="grafana"}').length).toBe(9);
});
it('returns the right position of a string in a stream selector', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
const nodePositions = getNodePositionsFromQuery('{job="grafana"}', [String]);
expect(nodePositions.length).toBe(1);
expect(nodePositions[0].from).toBe(5);
expect(nodePositions[0].to).toBe(14);
});
it('returns an empty array with a wrong expr', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
const nodePositions = getNodePositionsFromQuery('not loql', [String]);
expect(nodePositions.length).toBe(0);
});
});
describe('interpolateShardingSelector', () => {
let queries: LokiQuery[] = [];
beforeEach(() => {
queries = [
{
refId: 'A',
expr: '{job="grafana"}',
},
];
});
it('returns the original query when there are no shards to interpolate', () => {
expect(interpolateShardingSelector(queries, [])).toEqual(queries);
});
it('adds the empty shard to the query when -1 is passed', () => {
expect(interpolateShardingSelector(queries, [-1])).toEqual([
{
refId: 'A',
expr: '{job="grafana", __stream_shard__=""}',
},
]);
});
it('uses an equality filter when a single shard is passed', () => {
expect(interpolateShardingSelector(queries, [13])).toEqual([
{
refId: 'A',
expr: '{job="grafana", __stream_shard__="13"}',
},
]);
});
it('supports multiple shard values', () => {
expect(interpolateShardingSelector(queries, [1, 13, 667])).toEqual([
{
refId: 'A',
expr: '{job="grafana", __stream_shard__=~"1|13|667"}',
},
]);
});
describe('For metric queries', () => {
let queries: LokiQuery[] = [];
beforeEach(() => {
queries = [
{
refId: 'A',
expr: 'count_over_time({job="grafana"} [5m])',
},
];
});
it('returns the original query when there are no shards to interpolate', () => {
expect(interpolateShardingSelector(queries, [])).toEqual(queries);
});
it('adds the empty shard to the query when -1 is passed', () => {
expect(interpolateShardingSelector(queries, [-1])).toEqual([
{
refId: 'A',
expr: 'count_over_time({job="grafana", __stream_shard__=""} | drop __stream_shard__ [5m])',
},
]);
});
it('uses an equality filter when a single shard is passed', () => {
expect(interpolateShardingSelector(queries, [13])).toEqual([
{
refId: 'A',
expr: 'count_over_time({job="grafana", __stream_shard__="13"} | drop __stream_shard__ [5m])',
},
]);
});
it('supports multiple shard values', () => {
expect(interpolateShardingSelector(queries, [1, 13, 667])).toEqual([
{
refId: 'A',
expr: 'count_over_time({job="grafana", __stream_shard__=~"1|13|667"} | drop __stream_shard__ [5m])',
},
]);
});
it('supports multiple metric queries values', () => {
const queries = [
{
refId: 'A',
expr: 'count_over_time({job="grafana"} [5m]) + count_over_time({job="grafana"} [5m])',
},
];
expect(interpolateShardingSelector(queries, [1, 13, 667])).toEqual([
{
refId: 'A',
expr: 'count_over_time({job="grafana", __stream_shard__=~"1|13|667"} | drop __stream_shard__ [5m]) + count_over_time({job="grafana", __stream_shard__=~"1|13|667"} | drop __stream_shard__ [5m])',
},
]);
});
});
});
describe('requestSupportsSharding', () => {
it('supports log queries with Scan direction', () => {
expect(requestSupportsSharding([{ refId: 'A', expr: '{place="luna"}', direction: LokiQueryDirection.Scan }])).toBe(
true
);
});
it('declines log queries without Scan direction', () => {
expect(
requestSupportsSharding([{ refId: 'A', expr: '{place="luna"}', direction: LokiQueryDirection.Backward }])
).toBe(false);
});
it.each([
'count_over_time({place="luna"}[1m])',
'sum_over_time({place="luna"}[1m])',
'sum by (level) (count_over_time({place="luna"}[1m]))',
'sum by (level) (rate({place="luna"}[1m]))',
'sum(sum by (level) (avg_over_time({place="luna"}[1m])))',
'sum(rate({place="luna"}[1m]))',
])('allows supported metric queries', (expr: string) => {
expect(requestSupportsSharding([{ refId: 'A', expr }])).toBe(true);
});
it.each([
'avg_over_time({place="luna"}[1m])',
'avg(sum_over_time({place="luna"}[1m]))',
'avg(rate({place="luna"}[1m]))',
'count_over_time({place="luna"}[1m]) / count_over_time({place="luna"}[1m])',
'avg(sum by (level) (avg_over_time({place="luna"}[1m])))',
'sum(rate({place="luna"}[1m])) / sum(rate({place="luna"}[1m]))',
])('declines supported metric queries', (expr: string) => {
expect(requestSupportsSharding([{ refId: 'A', expr }])).toBe(false);
});
});