# Copyright (c) reifydb.com 2025
# This file is licensed under the Apache-2.0, see license.md file
query 'CALL rql::tokenize("from [ { id: 1, name: \'Zebra\' }, { id: 2, name: \'Apple\' }, { id: 3, name: \'中文\' }, { id: 4, name: \'العربية\' }, { id: 5, name: \'Banana\' } ] insert test::unicode_names")'
---
> +-------+--------+----------+--------------+-----------------+
> | idx | line | column | kind | value |
> +-------+--------+----------+--------------+-----------------+
> | 0 | 1 | 1 | Keyword | From |
> | 1 | 1 | 6 | Operator | OpenBracket |
> | 2 | 1 | 8 | Operator | OpenCurly |
> | 3 | 1 | 10 | Identifier | id |
> | 4 | 1 | 12 | Operator | Colon |
> | 5 | 1 | 14 | Literal | Number |
> | 6 | 1 | 15 | Separator | Comma |
> | 7 | 1 | 17 | Identifier | name |
> | 8 | 1 | 21 | Operator | Colon |
> | 9 | 1 | 24 | Literal | Text |
> | 10 | 1 | 31 | Operator | CloseCurly |
> | 11 | 1 | 32 | Separator | Comma |
> | 12 | 1 | 34 | Operator | OpenCurly |
> | 13 | 1 | 36 | Identifier | id |
> | 14 | 1 | 38 | Operator | Colon |
> | 15 | 1 | 40 | Literal | Number |
> | 16 | 1 | 41 | Separator | Comma |
> | 17 | 1 | 43 | Identifier | name |
> | 18 | 1 | 47 | Operator | Colon |
> | 19 | 1 | 50 | Literal | Text |
> | 20 | 1 | 57 | Operator | CloseCurly |
> | 21 | 1 | 58 | Separator | Comma |
> | 22 | 1 | 60 | Operator | OpenCurly |
> | 23 | 1 | 62 | Identifier | id |
> | 24 | 1 | 64 | Operator | Colon |
> | 25 | 1 | 66 | Literal | Number |
> | 26 | 1 | 67 | Separator | Comma |
> | 27 | 1 | 69 | Identifier | name |
> | 28 | 1 | 73 | Operator | Colon |
> | 29 | 1 | 76 | Literal | Text |
> | 30 | 1 | 80 | Operator | CloseCurly |
> | 31 | 1 | 81 | Separator | Comma |
> | 32 | 1 | 83 | Operator | OpenCurly |
> | 33 | 1 | 85 | Identifier | id |
> | 34 | 1 | 87 | Operator | Colon |
> | 35 | 1 | 89 | Literal | Number |
> | 36 | 1 | 90 | Separator | Comma |
> | 37 | 1 | 92 | Identifier | name |
> | 38 | 1 | 96 | Operator | Colon |
> | 39 | 1 | 99 | Literal | Text |
> | 40 | 1 | 108 | Operator | CloseCurly |
> | 41 | 1 | 109 | Separator | Comma |
> | 42 | 1 | 111 | Operator | OpenCurly |
> | 43 | 1 | 113 | Identifier | id |
> | 44 | 1 | 115 | Operator | Colon |
> | 45 | 1 | 117 | Literal | Number |
> | 46 | 1 | 118 | Separator | Comma |
> | 47 | 1 | 120 | Identifier | name |
> | 48 | 1 | 124 | Operator | Colon |
> | 49 | 1 | 127 | Literal | Text |
> | 50 | 1 | 135 | Operator | CloseCurly |
> | 51 | 1 | 137 | Operator | CloseBracket |
> | 52 | 1 | 139 | Keyword | Insert |
> | 53 | 1 | 146 | Identifier | test |
> | 54 | 1 | 150 | Operator | DoubleColon |
> | 55 | 1 | 152 | Identifier | unicode_names |
> +-------+--------+----------+--------------+-----------------+
>