tx7do/go-utils
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: query_parser + stringcase.

Browse Source
This commit is contained in:
Bobo
2025-06-23 23:26:59 +08:00
parent de36ab695d
commit 55c80024c2
22 changed files with 1362 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
query_parser/README.md Normal file
View File

@@ -0,0 +1,70 @@
# 查询解析器
## 排序规则
排序操作本质上是`SQL`里面的`Order By`条件。
| 序列 | 示例 | 备注 |
|----|--------------------|--------------|
| 升序 | `["type"]` | |
| 降序 | `["-create_time"]` | 字段名前加`-`是为降序 |
## 过滤规则
过滤器操作本质上是`SQL`里面的`WHERE`条件。
过滤器的规则，遵循了Python的ORM的规则，比如：
- [Tortoise ORM Filtering](https://tortoise.github.io/query.html#filtering)。
- [Django Field lookups](https://docs.djangoproject.com/en/4.2/ref/models/querysets/#field-lookups)
如果只是普通的查询，只需要传递`字段名`即可，但是如果需要一些特殊的查询，那么就需要加入`操作符`了。
特殊查询的语法规则其实很简单，就是使用双下划线`__`分割字段名和操作符：
```text
{字段名}__{查找类型} : {值}
{字段名}.{JSON字段名}__{查找类型} : {值}
```
| 查找类型 | 示例 | SQL | 备注 |
|-------------|---------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------|
| not | `{"name__not" : "tom"}` | `WHERE NOT ("name" = "tom")` | |
| in | `{"name__in" : "[\"tom\", \"jimmy\"]"}` | `WHERE name IN ("tom", "jimmy")` | |
| not_in | `{"name__not_in" : "[\"tom\", \"jimmy\"]"}` | `WHERE name NOT IN ("tom", "jimmy")` | |
| gte | `{"create_time__gte" : "2023-10-25"}` | `WHERE "create_time" >= "2023-10-25"` | |
| gt | `{"create_time__gt" : "2023-10-25"}` | `WHERE "create_time" > "2023-10-25"` | |
| lte | `{"create_time__lte" : "2023-10-25"}` | `WHERE "create_time" <= "2023-10-25"` | |
| lt | `{"create_time__lt" : "2023-10-25"}` | `WHERE "create_time" < "2023-10-25"` | |
| range | `{"create_time__range" : "[\"2023-10-25\", \"2024-10-25\"]"}` | `WHERE "create_time" BETWEEN "2023-10-25" AND "2024-10-25"` <br>或<br> `WHERE "create_time" >= "2023-10-25" AND "create_time" <= "2024-10-25"` | 需要注意的是: <br>1. 有些数据库的BETWEEN实现的开闭区间可能不一样。<br>2. 日期`2005-01-01`会被隐式转换为：`2005-01-01 00:00:00`，两个日期一致就会导致查询不到数据。 |
| isnull | `{"name__isnull" : "True"}` | `WHERE name IS NULL` | |
| not_isnull | `{"name__not_isnull" : "False"}` | `WHERE name IS NOT NULL` | |
| contains | `{"name__contains" : "L"}` | `WHERE name LIKE '%L%';` | |
| icontains | `{"name__icontains" : "L"}` | `WHERE name ILIKE '%L%';` | |
| startswith | `{"name__startswith" : "La"}` | `WHERE name LIKE 'La%';` | |
| istartswith | `{"name__istartswith" : "La"}` | `WHERE name ILIKE 'La%';` | |
| endswith | `{"name__endswith" : "a"}` | `WHERE name LIKE '%a';` | |
| iendswith | `{"name__iendswith" : "a"}` | `WHERE name ILIKE '%a';` | |
| exact | `{"name__exact" : "a"}` | `WHERE name LIKE 'a';` | |
| iexact | `{"name__iexact" : "a"}` | `WHERE name ILIKE 'a';` | |
| regex | `{"title__regex" : "^(An?\|The) +"}` | MySQL: `WHERE title REGEXP BINARY '^(An?\|The) +'` <br> Oracle: `WHERE REGEXP_LIKE(title, '^(An?\|The) +', 'c');` <br> PostgreSQL: `WHERE title ~ '^(An?\|The) +';` <br> SQLite: `WHERE title REGEXP '^(An?\|The) +';` | |
| iregex | `{"title__iregex" : "^(an?\|the) +"}` | MySQL: `WHERE title REGEXP '^(an?\|the) +'` <br> Oracle: `WHERE REGEXP_LIKE(title, '^(an?\|the) +', 'i');` <br> PostgreSQL: `WHERE title ~* '^(an?\|the) +';` <br> SQLite: `WHERE title REGEXP '(?i)^(an?\|the) +';` | |
| search | | | |
以及将日期提取出来的查找类型：
| 查找类型 | 示例 | SQL | 备注 |
|--------------|--------------------------------------|---------------------------------------------------|----------------------|
| date | `{"pub_date__date" : "2023-01-01"}` | `WHERE DATE(pub_date) = '2023-01-01'` | |
| year | `{"pub_date__year" : "2023"}` | `WHERE EXTRACT('YEAR' FROM pub_date) = '2023'` | 哪一年 |
| iso_year | `{"pub_date__iso_year" : "2023"}` | `WHERE EXTRACT('ISOYEAR' FROM pub_date) = '2023'` | ISO 8601 一年中的周数 |
| month | `{"pub_date__month" : "12"}` | `WHERE EXTRACT('MONTH' FROM pub_date) = '12'` | 月份，1-12 |
| day | `{"pub_date__day" : "3"}` | `WHERE EXTRACT('DAY' FROM pub_date) = '3'` | 该月的某天(1-31) |
| week | `{"pub_date__week" : "7"}` | `WHERE EXTRACT('WEEK' FROM pub_date) = '7'` | ISO 8601 周编号 一年中的周数 |
| week_day | `{"pub_date__week_day" : "tom"}` | `` | 星期几 |
| iso_week_day | `{"pub_date__iso_week_day" : "tom"}` | `` | |
| quarter | `{"pub_date__quarter" : "1"}` | `WHERE EXTRACT('QUARTER' FROM pub_date) = '1'` | 一年中的季度 |
| time | `{"pub_date__time" : "12:59:59"}` | `` | |
| hour | `{"pub_date__hour" : "12"}` | `WHERE EXTRACT('HOUR' FROM pub_date) = '12'` | 小时(0-23) |
| minute | `{"pub_date__minute" : "59"}` | `WHERE EXTRACT('MINUTE' FROM pub_date) = '59'` | 分钟 (0-59) |
| second | `{"pub_date__second" : "59"}` | `WHERE EXTRACT('SECOND' FROM pub_date) = '59'` | 秒 (0-59) |

126
query_parser/filter.go Normal file
View File

@@ -0,0 +1,126 @@
package query_parser
import (
"strings"
"github.com/go-kratos/kratos/v2/encoding"
_ "github.com/go-kratos/kratos/v2/encoding/json"
"github.com/tx7do/go-utils/stringcase"
)
type FilterOperator string
const (
FilterNot = "not" // 不等于
FilterIn = "in" // 检查值是否在列表中
FilterNotIn = "not_in" // 不在列表中
FilterGTE = "gte" // 大于或等于传递的值
FilterGT = "gt" // 大于传递值
FilterLTE = "lte" // 小于或等于传递值
FilterLT = "lt" // 小于传递值
FilterRange = "range" // 是否介于和给定的两个值之间
FilterIsNull = "isnull" // 是否为空
FilterNotIsNull = "not_isnull" // 是否不为空
FilterContains = "contains" // 是否包含指定的子字符串
FilterInsensitiveContains = "icontains" // 不区分大小写，是否包含指定的子字符串
FilterStartsWith = "startswith" // 以值开头
FilterInsensitiveStartsWith = "istartswith" // 不区分大小写，以值开头
FilterEndsWith = "endswith" // 以值结尾
FilterInsensitiveEndsWith = "iendswith" // 不区分大小写，以值结尾
FilterExact = "exact" // 精确匹配
FilterInsensitiveExact = "iexact" // 不区分大小写，精确匹配
FilterRegex = "regex" // 正则表达式
FilterInsensitiveRegex = "iregex" // 不区分大小写，正则表达式
FilterSearch = "search" // 全文搜索
)
const (
DatePartDate = "date" // 日期
DatePartYear = "year" // 年
DatePartISOYear = "iso_year" // ISO8601 一年中的周数
DatePartQuarter = "quarter" // 季度
DatePartMonth = "month" // 月
DatePartWeek = "week" // ISO8601 周编号 一年中的周数
DatePartWeekDay = "week_day" // 星期几
DatePartISOWeekDay = "iso_week_day" // ISO8601 星期几
DatePartDay = "day" // 日
DatePartTime = "time" // 小时：分钟：秒
DatePartHour = "hour" // 小时
DatePartMinute = "minute" // 分钟
DatePartSecond = "second" // 秒
DatePartMicrosecond = "microsecond" // 微秒
)
const (
QueryDelimiter = "__" // 分隔符
JsonFieldDelimiter = "." // JSONB字段分隔符
)
type FilterHandler func(field, operator, value string)
// ParseFilterJSONString 解析过滤条件的JSON字符串，调用处理函数
func ParseFilterJSONString(query string, handler FilterHandler) error {
if query == "" {
return nil
}
codec := encoding.GetCodec("json")
var err error
queryMap := make(map[string]string)
if err = codec.Unmarshal([]byte(query), &queryMap); err == nil {
for k, v := range queryMap {
ParseFilterField(k, v, handler)
}
return nil
}
var queryMapArray []map[string]string
if err = codec.Unmarshal([]byte(query), &queryMapArray); err == nil {
for _, item := range queryMapArray {
for k, v := range item {
ParseFilterField(k, v, handler)
}
}
return nil
}
return err
}
// ParseFilterField 解析过滤条件字符串，调用处理函数
func ParseFilterField(key, value string, handler FilterHandler) {
if key == "" || value == "" {
return // 没有过滤条件
}
// 处理字段和操作符
parts := SplitFieldAndOperator(key)
if len(parts) < 1 {
return // 无效的字段格式
}
field := strings.TrimSpace(parts[0])
if field == "" {
return
}
field = stringcase.ToSnakeCase(parts[0])
op := ""
if len(parts) > 1 {
op = parts[1]
}
handler(field, op, value)
}
// SplitFieldAndOperator 将字段字符串按分隔符分割成字段名和操作符
func SplitFieldAndOperator(field string) []string {
return strings.Split(field, QueryDelimiter)
}
// SplitJSONField 将JSONB字段字符串按分隔符分割成多个字段
func SplitJSONField(field string) []string {
return strings.Split(field, JsonFieldDelimiter)
}

181
query_parser/filter_test.go Normal file
View File

@@ -0,0 +1,181 @@
package query_parser
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestParseFilterJSONString(t *testing.T) {
var results []struct {
Field string
Operator string
Value string
}
handler := func(field, operator, value string) {
results = append(results, struct {
Field string
Operator string
Value string
}{Field: field, Operator: operator, Value: value})
}
// 测试解析单个过滤条件
results = nil
err := ParseFilterJSONString(`{"name__exact":"John"}`, handler)
assert.NoError(t, err)
assert.Equal(t, 1, len(results))
assert.Equal(t, "name", results[0].Field)
assert.Equal(t, "exact", results[0].Operator)
assert.Equal(t, "John", results[0].Value)
// 测试解析多个过滤条件
results = nil
err = ParseFilterJSONString(`[{"age__gte":"30"},{"status__exact":"active"}]`, handler)
assert.NoError(t, err)
assert.Equal(t, 2, len(results))
assert.Equal(t, "age", results[0].Field)
assert.Equal(t, "gte", results[0].Operator)
assert.Equal(t, "30", results[0].Value)
assert.Equal(t, "status", results[1].Field)
assert.Equal(t, "exact", results[1].Operator)
assert.Equal(t, "active", results[1].Value)
// 测试空字符串
results = nil
err = ParseFilterJSONString("", handler)
assert.NoError(t, err)
assert.Equal(t, 0, len(results))
// 测试无效的JSON字符串
results = nil
err = ParseFilterJSONString(`invalid_json`, handler)
assert.Error(t, err)
assert.Equal(t, 0, len(results))
// 测试包含特殊字符的字段和值
results = nil
err = ParseFilterJSONString(`{"na!me__exact":"Jo@hn"}`, handler)
assert.NoError(t, err)
assert.Equal(t, 1, len(results))
assert.Equal(t, "na!me", results[0].Field)
assert.Equal(t, "exact", results[0].Operator)
assert.Equal(t, "Jo@hn", results[0].Value)
// 测试包含特殊字符的多个过滤条件
results = nil
err = ParseFilterJSONString(`[{"ag#e__gte":"30"},{"sta$tus__exact":"ac^tive"}]`, handler)
assert.NoError(t, err)
assert.Equal(t, 2, len(results))
assert.Equal(t, "ag#e", results[0].Field)
assert.Equal(t, "gte", results[0].Operator)
assert.Equal(t, "30", results[0].Value)
assert.Equal(t, "sta$tus", results[1].Field)
assert.Equal(t, "exact", results[1].Operator)
assert.Equal(t, "ac^tive", results[1].Value)
// 测试包含特殊字符的无效 JSON 字符串
results = nil
err = ParseFilterJSONString(`{"na!me__exact":Jo@hn}`, handler)
assert.Error(t, err)
assert.Equal(t, 0, len(results))
}
func TestParseFilterField(t *testing.T) {
var results []struct {
Field string
Operator string
Value string
}
handler := func(field, operator, value string) {
results = append(results, struct {
Field string
Operator string
Value string
}{Field: field, Operator: operator, Value: value})
}
// 测试正常解析
results = nil
ParseFilterField("name__exact", "John", handler)
assert.Equal(t, 1, len(results))
assert.Equal(t, "name", results[0].Field)
assert.Equal(t, "exact", results[0].Operator)
assert.Equal(t, "John", results[0].Value)
// 测试无操作符解析
results = nil
ParseFilterField("name", "John", handler)
assert.Equal(t, 1, len(results))
assert.Equal(t, "name", results[0].Field)
assert.Equal(t, "", results[0].Operator)
assert.Equal(t, "John", results[0].Value)
// 测试空字段
results = nil
ParseFilterField("", "John", handler)
assert.Equal(t, 0, len(results))
// 测试空值
results = nil
ParseFilterField("name__exact", "", handler)
assert.Equal(t, 0, len(results))
// 测试无效字段格式
results = nil
ParseFilterField("__exact", "John", handler)
assert.Equal(t, 0, len(results))
}
func TestSplitFieldAndOperator(t *testing.T) {
// 测试正常分割
result := SplitFieldAndOperator("name__exact")
assert.Equal(t, 2, len(result))
assert.Equal(t, "name", result[0])
assert.Equal(t, "exact", result[1])
// 测试无操作符
result = SplitFieldAndOperator("name")
assert.Equal(t, 1, len(result))
assert.Equal(t, "name", result[0])
// 测试空字符串
result = SplitFieldAndOperator("")
assert.Equal(t, 1, len(result))
assert.Equal(t, "", result[0])
// 测试多个分隔符
result = SplitFieldAndOperator("name__exact__extra")
assert.Equal(t, 3, len(result))
assert.Equal(t, "name", result[0])
assert.Equal(t, "exact", result[1])
assert.Equal(t, "extra", result[2])
}
func TestSplitJSONField(t *testing.T) {
// 测试正常分割
result := SplitJSONField("user.address.city")
assert.Equal(t, 3, len(result))
assert.Equal(t, "user", result[0])
assert.Equal(t, "address", result[1])
assert.Equal(t, "city", result[2])
// 测试单个字段
result = SplitJSONField("user")
assert.Equal(t, 1, len(result))
assert.Equal(t, "user", result[0])
// 测试空字符串
result = SplitJSONField("")
assert.Equal(t, 1, len(result))
assert.Equal(t, "", result[0])
// 测试多个分隔符
result = SplitJSONField("user..address.city")
assert.Equal(t, 4, len(result))
assert.Equal(t, "user", result[0])
assert.Equal(t, "", result[1])
assert.Equal(t, "address", result[2])
assert.Equal(t, "city", result[3])
}

22
query_parser/go.mod Normal file
View File

@@ -0,0 +1,22 @@
module github.com/tx7do/go-utils/query_parser
go 1.23.0
toolchain go1.23.2
require (
github.com/go-kratos/kratos/v2 v2.8.4
github.com/stretchr/testify v1.10.0
github.com/tx7do/go-utils v1.1.28
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
replace github.com/tx7do/go-utils => ../

50
query_parser/orderby.go Normal file
View File

@@ -0,0 +1,50 @@
package query_parser
import (
"strings"
)
type OrderByHandler func(field string, desc bool)
// ParseOrderByString 解析排序字符串，调用处理函数。
func ParseOrderByString(orderBy string, handler OrderByHandler) error {
if orderBy == "" {
return nil
}
parts := strings.Split(orderBy, ",")
for _, part := range parts {
part = strings.TrimSpace(part)
ParseOrderByField(part, handler)
}
return nil
}
// ParseOrderByStrings 解析多个排序字符串，调用处理函数。
func ParseOrderByStrings(orderBys []string, handler OrderByHandler) error {
for _, v := range orderBys {
if v == "" {
continue
}
ParseOrderByField(v, handler)
}
return nil
}
// ParseOrderByField 解析单个排序字段，调用处理函数。
func ParseOrderByField(orderBy string, handler OrderByHandler) {
orderBy = strings.TrimSpace(orderBy)
if orderBy == "" {
return // 没有排序条件
}
if strings.HasPrefix(orderBy, "-") {
handler(orderBy[1:], true) // 降序
} else if strings.HasPrefix(orderBy, "+") {
handler(orderBy[1:], false) // 升序
} else {
handler(orderBy, false) // 升序
}
}

126
query_parser/orderby_test.go Normal file
View File

@@ -0,0 +1,126 @@
package query_parser
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseOrderByString(t *testing.T) {
var results []struct {
Field string
Desc bool
}
handler := func(field string, desc bool) {
results = append(results, struct {
Field string
Desc bool
}{Field: field, Desc: desc})
}
// 测试正常解析
err := ParseOrderByString("name,-age,+created_at", handler)
assert.NoError(t, err)
assert.Equal(t, 3, len(results))
assert.Equal(t, "name", results[0].Field)
assert.False(t, results[0].Desc)
assert.Equal(t, "age", results[1].Field)
assert.True(t, results[1].Desc)
assert.Equal(t, "created_at", results[2].Field)
assert.False(t, results[2].Desc)
// 测试空字符串
results = nil
err = ParseOrderByString("", handler)
assert.NoError(t, err)
assert.Equal(t, 0, len(results))
// 测试只有空格的字符串
results = nil
err = ParseOrderByString(" ", handler)
assert.NoError(t, err)
assert.Equal(t, 0, len(results))
}
func TestParseOrderByStrings(t *testing.T) {
var results []struct {
Field string
Desc bool
}
handler := func(field string, desc bool) {
results = append(results, struct {
Field string
Desc bool
}{Field: field, Desc: desc})
}
// 测试正常解析
err := ParseOrderByStrings([]string{"name", "-age", "+created_at"}, handler)
assert.NoError(t, err)
assert.Equal(t, 3, len(results))
assert.Equal(t, "name", results[0].Field)
assert.False(t, results[0].Desc)
assert.Equal(t, "age", results[1].Field)
assert.True(t, results[1].Desc)
assert.Equal(t, "created_at", results[2].Field)
assert.False(t, results[2].Desc)
// 测试空字符串数组
results = nil
err = ParseOrderByStrings([]string{}, handler)
assert.NoError(t, err)
assert.Equal(t, 0, len(results))
// 测试包含空字符串的数组
results = nil
err = ParseOrderByStrings([]string{"", " "}, handler)
assert.NoError(t, err)
assert.Equal(t, 0, len(results))
}
func TestParseOrderByField(t *testing.T) {
var results []struct {
Field string
Desc bool
}
handler := func(field string, desc bool) {
results = append(results, struct {
Field string
Desc bool
}{Field: field, Desc: desc})
}
// 测试升序解析
results = nil
ParseOrderByField("name", handler)
assert.Equal(t, 1, len(results))
assert.Equal(t, "name", results[0].Field)
assert.False(t, results[0].Desc)
// 测试降序解析
results = nil
ParseOrderByField("-age", handler)
assert.Equal(t, 1, len(results))
assert.Equal(t, "age", results[0].Field)
assert.True(t, results[0].Desc)
// 测试带+的升序解析
results = nil
ParseOrderByField("+created_at", handler)
assert.Equal(t, 1, len(results))
assert.Equal(t, "created_at", results[0].Field)
assert.False(t, results[0].Desc)
// 测试空字符串
results = nil
ParseOrderByField("", handler)
assert.Equal(t, 0, len(results))
// 测试只有空格的字符串
results = nil
ParseOrderByField(" ", handler)
assert.Equal(t, 0, len(results))
}