Add functions in transform plugin to cover modify plugin

plugin/README.md

@@ -788,7 +788,7 @@ pipelines:
     - type: transform
       source: |
         # parse lines like "INFO 2025-05-25 11:11:11,222 [shard 1] compaction - done"
-        m = capture(.log, r'^(?P<level>\S+)\s+(?P<time>\S+ \S+)\s+\[(?P<shard>[^\]]+)\]\s+(?P<operation>\S+)\s+-\s+(?P<message>.+)$')
+        m = parse_regex(.log, r'^(?P<level>\S+)\s+(?P<time>\S+ \S+)\s+\[(?P<shard>[^\]]+)\]\s+(?P<operation>\S+)\s+-\s+(?P<message>.+)$')
         if m != null {
           .level = m.level
           .time = m.time

plugin/action/README.md

@@ -609,7 +609,7 @@ pipelines:
     - type: transform
       source: |
         # parse lines like "INFO 2025-05-25 11:11:11,222 [shard 1] compaction - done"
-        m = capture(.log, r'^(?P<level>\S+)\s+(?P<time>\S+ \S+)\s+\[(?P<shard>[^\]]+)\]\s+(?P<operation>\S+)\s+-\s+(?P<message>.+)$')
+        m = parse_regex(.log, r'^(?P<level>\S+)\s+(?P<time>\S+ \S+)\s+\[(?P<shard>[^\]]+)\]\s+(?P<operation>\S+)\s+-\s+(?P<message>.+)$')
         if m != null {
           .level = m.level
           .time = m.time

plugin/action/transform/README.idoc.md

@@ -61,9 +61,9 @@ In conditions `null` and `false` are falsy; every other value is truthy.
 Variables hold intermediate values and live for one event:
 
 ```
-name = .user.name             # read a field into a variable
-parts = capture(.log, r'...') # keep a function result
-.out = name                   # write it back to the event
+name = .user.name                 # read a field into a variable
+parts = parse_regex(.log, r'...')  # keep a function result
+.out = name                       # write it back to the event
 ```
 
 Fields of object values are accessed with a dot or an index; both forms are
@@ -95,7 +95,7 @@ In order of increasing precedence:
 | `f()` `a[i]` `a.b` | call, index, member access |
 
 `+` concatenates only strings with strings — convert other values first:
-`"code " + string(.code)`.
+`"code " + to_string(.code)`.
 
 ### Control flow
 
@@ -129,22 +129,32 @@ are required; named arguments are optional and fall back to their defaults:
   .level = upcase(.level)    # "info" -> "INFO"
   ```
 
-+ `string(value)` — converts any value to its string representation; `null`
++ `to_string(value)` — converts any value to its string representation; `null`
   becomes an empty string. Use it to build strings from non-string fields:
   ```
-  .msg = "code is " + string(.code)
+  .msg = "code is " + to_string(.code)
   ```
 
-+ `capture(value, pattern)` — matches the string against a regular expression
-  and returns an object of its named groups `(?P<name>...)`, or `null` when the
-  value does not match (unnamed groups are ignored):
++ `parse_regex(value, pattern, numeric_groups: false)` — matches the string
+  against a regular expression and returns an object of its named groups
+  `(?P<name>...)`, or `null` when the value does not match (unnamed groups are
+  ignored):
   ```
-  m = capture(.log, r'^(?P<level>\S+)\s+(?P<message>.+)$')
+  m = parse_regex(.log, r'^(?P<level>\S+)\s+(?P<message>.+)$')
   if m != null {
     .level = m.level
     .message = m.message
   }
   ```
+  With `numeric_groups: true` every group is additionally keyed by its index as a
+  string — `"0"` is the whole match — which is how a pattern without named groups
+  is read:
+  ```
+  m = parse_regex(.message, r'(\w+):.*', numeric_groups: true)
+  if m != null {
+    .level = m["1"]
+  }
+  ```
 
 + `after(value, separator)` — returns everything after the first occurrence of
   `separator`; the value is returned unchanged when the separator is not found.
@@ -164,6 +174,47 @@ are required; named arguments are optional and fall back to their defaults:
   .shard = between(.log, "[", "]")
   ```
 
++ `parse_regex_all(value, pattern, group: 0, limit: -1)` — returns every match of
+  `pattern` as an array, or an empty array when nothing matches. `group` selects
+  a capture group (`0` is the whole match) and a negative `limit` collects all
+  occurrences:
+  ```
+  .ids = parse_regex_all(.log, r'id=(\w+)', group: 1)
+  ```
+
++ `join(value, separator)` — joins an array of strings into one string. Only
+  strings are joined; convert other values with `to_string()` first:
+  ```
+  .extracted = join(parse_regex_all(.message, r're\d+', limit: 2), ",")   # "re1,re2"
+  ```
+
++ `trim(value, cutset)`, `trim_left(value, cutset)`,
+  `trim_right(value, cutset)` — strip characters from both ends, the start or
+  the end. `cutset` is a *set of characters*, not a substring: `trim_right(v, "ms")`
+  removes every trailing `m` and `s`.
+  ```
+  .message = trim_right(.message, "\n")
+  ```
+
++ `trim_to(value, cutset)`, `trim_to_left(value, cutset)`,
+  `trim_to_right(value, cutset)` — cut the string back to a delimiter, keeping the
+  delimiter itself; the value is returned unchanged when it is not found. Here
+  `cutset` is a *substring*: the left side trims to its first occurrence, the
+  right side to its last one, so the outermost pair is kept:
+  ```
+  # 'some data {"took":"200ms"} some data' -> '{"took":"200ms"}'
+  .message = trim_to_right(trim_to_left(.message, "{"), "}")
+  ```
+
++ `slice(value, start, end: null)` — returns the part of the string between two
+  positions, counted in characters. Both positions may be negative to count from
+  the end, and positions outside the string are clamped rather than raising an
+  error. `end` defaults to the end of the string:
+  ```
+  .head = slice(.message, 0, end: 10)   # first 10 characters
+  .tail = slice(.message, -5)           # last 5 characters
+  ```
+
 + `lookup(value, table, default: <unchanged>)` — translates a value through a
   table of replacements. It turns enumeration codes into readable names without
   a chain of `if`s:

plugin/action/transform/README.md

@@ -12,7 +12,7 @@ pipelines:
     - type: transform
       source: |
         # parse lines like "INFO 2025-05-25 11:11:11,222 [shard 1] compaction - done"
-        m = capture(.log, r'^(?P<level>\S+)\s+(?P<time>\S+ \S+)\s+\[(?P<shard>[^\]]+)\]\s+(?P<operation>\S+)\s+-\s+(?P<message>.+)$')
+        m = parse_regex(.log, r'^(?P<level>\S+)\s+(?P<time>\S+ \S+)\s+\[(?P<shard>[^\]]+)\]\s+(?P<operation>\S+)\s+-\s+(?P<message>.+)$')
         if m != null {
           .level = m.level
           .time = m.time
@@ -106,9 +106,9 @@ In conditions `null` and `false` are falsy; every other value is truthy.
 Variables hold intermediate values and live for one event:
 
 ```
-name = .user.name             # read a field into a variable
-parts = capture(.log, r'...') # keep a function result
-.out = name                   # write it back to the event
+name = .user.name                 # read a field into a variable
+parts = parse_regex(.log, r'...')  # keep a function result
+.out = name                       # write it back to the event
 ```
 
 Fields of object values are accessed with a dot or an index; both forms are
@@ -140,7 +140,7 @@ In order of increasing precedence:
 | `f()` `a[i]` `a.b` | call, index, member access |
 
 `+` concatenates only strings with strings — convert other values first:
-`"code " + string(.code)`.
+`"code " + to_string(.code)`.
 
 ### Control flow
 
@@ -174,22 +174,32 @@ are required; named arguments are optional and fall back to their defaults:
   .level = upcase(.level)    # "info" -> "INFO"
   ```
 
-+ `string(value)` — converts any value to its string representation; `null`
++ `to_string(value)` — converts any value to its string representation; `null`
   becomes an empty string. Use it to build strings from non-string fields:
   ```
-  .msg = "code is " + string(.code)
+  .msg = "code is " + to_string(.code)
   ```
 
-+ `capture(value, pattern)` — matches the string against a regular expression
-  and returns an object of its named groups `(?P<name>...)`, or `null` when the
-  value does not match (unnamed groups are ignored):
++ `parse_regex(value, pattern, numeric_groups: false)` — matches the string
+  against a regular expression and returns an object of its named groups
+  `(?P<name>...)`, or `null` when the value does not match (unnamed groups are
+  ignored):
   ```
-  m = capture(.log, r'^(?P<level>\S+)\s+(?P<message>.+)$')
+  m = parse_regex(.log, r'^(?P<level>\S+)\s+(?P<message>.+)$')
   if m != null {
     .level = m.level
     .message = m.message
   }
   ```
+  With `numeric_groups: true` every group is additionally keyed by its index as a
+  string — `"0"` is the whole match — which is how a pattern without named groups
+  is read:
+  ```
+  m = parse_regex(.message, r'(\w+):.*', numeric_groups: true)
+  if m != null {
+    .level = m["1"]
+  }
+  ```
 
 + `after(value, separator)` — returns everything after the first occurrence of
   `separator`; the value is returned unchanged when the separator is not found.
@@ -209,6 +219,47 @@ are required; named arguments are optional and fall back to their defaults:
   .shard = between(.log, "[", "]")
   ```
 
++ `parse_regex_all(value, pattern, group: 0, limit: -1)` — returns every match of
+  `pattern` as an array, or an empty array when nothing matches. `group` selects
+  a capture group (`0` is the whole match) and a negative `limit` collects all
+  occurrences:
+  ```
+  .ids = parse_regex_all(.log, r'id=(\w+)', group: 1)
+  ```
+
++ `join(value, separator)` — joins an array of strings into one string. Only
+  strings are joined; convert other values with `to_string()` first:
+  ```
+  .extracted = join(parse_regex_all(.message, r're\d+', limit: 2), ",")   # "re1,re2"
+  ```
+
++ `trim(value, cutset)`, `trim_left(value, cutset)`,
+  `trim_right(value, cutset)` — strip characters from both ends, the start or
+  the end. `cutset` is a *set of characters*, not a substring: `trim_right(v, "ms")`
+  removes every trailing `m` and `s`.
+  ```
+  .message = trim_right(.message, "\n")
+  ```
+
++ `trim_to(value, cutset)`, `trim_to_left(value, cutset)`,
+  `trim_to_right(value, cutset)` — cut the string back to a delimiter, keeping the
+  delimiter itself; the value is returned unchanged when it is not found. Here
+  `cutset` is a *substring*: the left side trims to its first occurrence, the
+  right side to its last one, so the outermost pair is kept:
+  ```
+  # 'some data {"took":"200ms"} some data' -> '{"took":"200ms"}'
+  .message = trim_to_right(trim_to_left(.message, "{"), "}")
+  ```
+
++ `slice(value, start, end: null)` — returns the part of the string between two
+  positions, counted in characters. Both positions may be negative to count from
+  the end, and positions outside the string are clamped rather than raising an
+  error. `end` defaults to the end of the string:
+  ```
+  .head = slice(.message, 0, end: 10)   # first 10 characters
+  .tail = slice(.message, -5)           # last 5 characters
+  ```
+
 + `lookup(value, table, default: <unchanged>)` — translates a value through a
   table of replacements. It turns enumeration codes into readable names without
   a chain of `if`s:

plugin/action/transform/compiler/fold_test.go

@@ -130,11 +130,11 @@ Assign
 		},
 		{
 			name: "regex literal folds after it is compiled",
-			src:  `m = capture(.log, r'(?P<a>\d+)')`,
+			src:  `m = parse_regex(.log, r'(?P<a>\d+)')`,
 			want: `
 Assign
   Ident(m)
-  Call(capture)
+  Call(parse_regex)
     Path(.log)
     Const(r'(?P<a>\d+)')`,
 		},

plugin/action/transform/core/ast.go

@@ -176,7 +176,7 @@ type ForExpr struct {
 // dump of a folded constant is stable across runs.
 //
 // Value.String() deliberately does not sort: it sits on the runtime path of
-// string(value), and debug output is the only place that needs a stable order.
+// to_string(value), and debug output is the only place that needs a stable order
 func dumpValue(v Value) string {
 	switch t := v.(type) {
 	case StringValue:

plugin/action/transform/core/value.go

@@ -208,7 +208,7 @@ func (v ArrayValue) String() string {
 }
 
 // Key order follows Go map iteration and is therefore not stable. This is on
-// the runtime path of string(value), so it does not pay for a sort; debug
+// the runtime path of to_string(value), so it does not pay for a sort; debug
 // output that needs a stable rendering uses dumpValue instead.
 func (v ObjectValue) String() string {
 	parts := make([]string, 0, len(v.V))

plugin/action/transform/core/value_test.go

@@ -10,7 +10,7 @@ import (
 
 // Debug output must not depend on Go map iteration order, so that the dump of a
 // folded constant is reproducible. Value.String() is deliberately unsorted -
-// it sits on the runtime path of string(value) - so the sort lives in dumpValue.
+// it sits on the runtime path of to_string(value) - so the sort lives in dumpValue
 func TestDumpConstIsStable(t *testing.T) {
 	t.Parallel()
 

plugin/action/transform/doc_examples_test.go

@@ -1,18 +1,49 @@
 package transform
 
 import (
+	"fmt"
+	"os"
+	"strings"
 	"testing"
 
 	"github.com/ozontech/file.d/plugin/action/transform/compiler"
 	"github.com/ozontech/file.d/plugin/action/transform/stdlib"
 	"github.com/stretchr/testify/require"
 )
 
+// validate examples from both the documentation source and the generated README
+func TestReadmeExamplesCompile(t *testing.T) {
+	for _, path := range []string{"README.idoc.md", "README.md"} {
+		t.Run(path, func(t *testing.T) {
+			data, err := os.ReadFile(path)
+			require.NoError(t, err)
+
+			blocks := strings.Split(string(data), "```")
+			count := 0
+			for i := 1; i < len(blocks); i += 2 {
+				// unlabelled code blocks contain transform programs
+				if !strings.HasPrefix(blocks[i], "\n") {
+					continue
+				}
+				count++
+				t.Run(fmt.Sprintf("example_%d", count), func(t *testing.T) {
+					cmp, err := compiler.NewCompiler(blocks[i])
+					require.NoError(t, err)
+					exprs, err := cmp.Compile()
+					require.NoError(t, err)
+					require.NoError(t, compiler.ValidateCalls(exprs, stdlib.GetRegistry()))
+				})
+			}
+			require.Positive(t, count, "documentation must contain program examples")
+		})
+	}
+}
+
 // compile and validate every program snippet shown in the plugin documentation.
 func TestDocExamplesCompile(t *testing.T) {
 	snippets := []string{
 		// introduction example
-		`m = capture(.log, r'^(?P<level>\S+)\s+(?P<time>\S+ \S+)\s+\[(?P<shard>[^\]]+)\]\s+(?P<operation>\S+)\s+-\s+(?P<message>.+)$')
+		`m = parse_regex(.log, r'^(?P<level>\S+)\s+(?P<time>\S+ \S+)\s+\[(?P<shard>[^\]]+)\]\s+(?P<operation>\S+)\s+-\s+(?P<message>.+)$')
 		if m != null {
 		  .level = m.level
 		  .time = m.time
@@ -27,11 +58,11 @@ func TestDocExamplesCompile(t *testing.T) {
 		`.a.b.c = 1`,
 		`del .user.password`,
 		// variables
-		"name = .user.name\nparts = capture(.log, r'...')\n.out = name",
+		"name = .user.name\nparts = parse_regex(.log, r'...')\n.out = name",
 		"x = parts.level\narr = [1]\narr[0] = 1\nobj = {}\nobj.key = \"value\"",
 		`a = b = 1`,
 		// operators
-		`.msg = "code is " + string(.code)`,
+		`.msg = "code is " + to_string(.code)`,
 		// control flow
 		`if .status >= 500 {
 		  .severity = "crit"
@@ -50,14 +81,24 @@ func TestDocExamplesCompile(t *testing.T) {
 		}`,
 		// functions
 		`.level = upcase(.level)`,
-		`m = capture(.log, r'^(?P<level>\S+)\s+(?P<message>.+)$')
+		`m = parse_regex(.log, r'^(?P<level>\S+)\s+(?P<message>.+)$')
 		if m != null {
 		  .level = m.level
 		  .message = m.message
 		}`,
 		`.message = after(.log, " - ")`,
 		`.level = before(.log, " ")`,
 		`.shard = between(.log, "[", "]")`,
+		`m = parse_regex(.message, r'(\w+):.*', numeric_groups: true)
+		if m != null {
+		  .level = m["1"]
+		}`,
+		`.ids = parse_regex_all(.log, r'id=(\w+)', group: 1)`,
+		`.extracted = join(parse_regex_all(.message, r're\d+', limit: 2), ",")`,
+		`.message = trim_right(.message, "\n")`,
+		`.message = trim_to_right(trim_to_left(.message, "{"), "}")`,
+		`.head = slice(.message, 0, end: 10)
+		.tail = slice(.message, -5)`,
 		`api_key = {"0": "produce", "1": "fetch", "2": "offsets"}
 		.kafka_request_api_key = lookup(.kafka_request_api_key, api_key)`,
 		`.severity = lookup(.status, {"500": "crit", "400": "warn"}, default: "ok")`,