tx7do/kratos-bootstrap
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: elasticsearch, influxdb.

Browse Source
This commit is contained in:
Bobo
2025-06-25 19:50:51 +08:00
parent 735ca567bf
commit d31ab9cdf3
16 changed files with 3217 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
database/elasticsearch/README.md Normal file
View File

@@ -0,0 +1,55 @@
# ElasticSearch
## 概念对比
| ES存储结构 | RDBMS存储结构 |
|----------|-----------|
| Index | 表 |
| Document | 行 |
| Field | 表字段 |
| Mapping | 表结构定义 |
## mapping
- 动态映射（dynamic mapping）
- 显式映射（explicit mapping）
- 严格映射（strict mappings）
## Docker部署
### 拉取镜像
```bash
docker pull bitnami/elasticsearch:latest
```
### 启动容器
```bash
docker run -itd \
--name elasticsearch \
-p 9200:9200 \
-p 9300:9300 \
-e ELASTICSEARCH_USERNAME=elastic \
-e ELASTICSEARCH_PASSWORD=elastic \
-e xpack.security.enabled=true \
-e discovery.type=single-node \
-e http.cors.enabled=true \
-e http.cors.allow-origin=http://localhost:13580,http://127.0.0.1:13580 \
-e http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization \
-e http.cors.allow-credentials=true \
bitnami/elasticsearch:latest
```
安装管理工具：
```bash
docker pull appbaseio/dejavu:latest
docker run -itd \
--name dejavu-test \
-p 13580:1358 \
appbaseio/dejavu:latest
```
访问管理工具：<http://localhost:13580/>

446
database/elasticsearch/client.go Normal file
View File

@@ -0,0 +1,446 @@
package elasticsearch
import (
"bytes"
"context"
"encoding/json"
"io"
"github.com/go-kratos/kratos/v2/encoding"
_ "github.com/go-kratos/kratos/v2/encoding/json"
"github.com/go-kratos/kratos/v2/log"
elasticsearchV9 "github.com/elastic/go-elasticsearch/v9"
esapiV9 "github.com/elastic/go-elasticsearch/v9/esapi"
conf "github.com/tx7do/kratos-bootstrap/api/gen/go/conf/v1"
pagination "github.com/tx7do/kratos-bootstrap/api/gen/go/pagination/v1"
)
type Client struct {
cli *elasticsearchV9.Client
log *log.Helper
codec encoding.Codec
}
func NewClient(logger log.Logger, cfg *conf.Bootstrap) (*Client, error) {
c := &Client{
log: log.NewHelper(log.With(logger, "module", "elasticsearch-client")),
codec: encoding.GetCodec("json"),
}
if err := c.createESClient(cfg); err != nil {
return nil, err
}
return c, nil
}
// createESClient 创建Elasticsearch客户端
func (c *Client) createESClient(cfg *conf.Bootstrap) error {
if cfg.Data == nil || cfg.Data.ElasticSearch == nil {
return nil // No Elasticsearch configuration provided
}
cli, err := elasticsearchV9.NewClient(
elasticsearchV9.Config{
Addresses: cfg.Data.ElasticSearch.GetAddresses(),
Username: cfg.Data.ElasticSearch.GetUsername(),
Password: cfg.Data.ElasticSearch.GetPassword(),
},
)
if err != nil {
c.log.Errorf("failed to create elasticsearch client: %v", err)
return err
}
c.cli = cli
return nil
}
func (c *Client) Close() {
}
// CheckConnectStatus 检查Elasticsearch连接
func (c *Client) CheckConnectStatus() bool {
if c.cli == nil {
return false
}
resp, err := c.cli.Info()
if err != nil {
c.log.Errorf("failed to connect to elasticsearch: %v", err)
return false
}
defer func(Body io.ReadCloser) {
if err = Body.Close(); err != nil {
c.log.Errorf("failed to close response body: %v", err)
}
}(resp.Body)
if resp.IsError() {
c.log.Errorf("Error: %s", resp.String())
return false
}
var r map[string]interface{}
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
log.Fatalf("Error parsing the response body: %s", err)
return false
}
c.log.Infof("Client Version: %s", elasticsearchV9.Version)
c.log.Infof("Server Version: %s", r["version"].(map[string]interface{})["number"])
return true
}
// IndexExists 检查索引是否存在
func (c *Client) IndexExists(ctx context.Context, indexName string) (bool, error) {
resp, err := c.cli.Indices.Exists(
[]string{indexName},
c.cli.Indices.Exists.WithContext(ctx),
)
if err != nil {
c.log.Errorf("failed to check if index exists: %v", err)
return false, err
}
return !resp.IsError(), nil
}
// CreateIndex 创建一条索引
//
// 如果mapping为空("")则表示不创建模型
func (c *Client) CreateIndex(ctx context.Context, indexName string, mapping, settings string) error {
exist, err := c.IndexExists(ctx, indexName)
if err != nil {
return err
}
if exist {
return ErrIndexAlreadyExists
}
body, err := MergeOptions(mapping, settings)
if err != nil {
c.log.Errorf("failed to merge options: %v", err)
return err
}
resp, err := c.cli.Indices.Create(
indexName,
c.cli.Indices.Create.WithContext(ctx),
c.cli.Indices.Create.WithBody(bytes.NewReader([]byte(body))),
)
if err != nil {
c.log.Errorf("failed to create index: %v", err)
return err
}
if resp.IsError() {
var errResp *ErrorResponse
if errResp, err = ParseErrorMessage(resp.Body); err != nil {
c.log.Errorf("failed to parse error message: %v", err)
return err
}
c.log.Errorf("create index failed: %s", errResp.Error)
return ErrCreateIndex
}
return nil
}
// DeleteIndex 删除一条索引
func (c *Client) DeleteIndex(ctx context.Context, indexName string) error {
exist, err := c.IndexExists(ctx, indexName)
if err != nil {
c.log.Errorf("failed to check if index exists: %v", err)
return err
}
if !exist {
return ErrIndexNotFound
}
resp, err := c.cli.Indices.Delete(
[]string{indexName},
c.cli.Indices.Delete.WithContext(ctx),
)
if err != nil {
c.log.Errorf("failed to delete index: %v", err)
return err
}
if resp.IsError() {
var errResp *ErrorResponse
if errResp, err = ParseErrorMessage(resp.Body); err != nil {
c.log.Errorf("failed to parse error message: %v", err)
return err
}
c.log.Errorf("delete index failed: %s", errResp.Error.Reason)
return ErrDeleteIndex
}
return nil
}
// DeleteDocument 删除一条数据
func (c *Client) DeleteDocument(ctx context.Context, indexName, id string) error {
_, err := c.cli.Delete(
indexName, id,
c.cli.Delete.WithContext(ctx),
)
if err != nil {
c.log.Errorf("failed to delete document: %v", err)
return err
}
return nil
}
// InsertDocument 插入一条数据
func (c *Client) InsertDocument(ctx context.Context, indexName, id string, data interface{}) error {
var err error
var dataBytes []byte
dataBytes, err = json.Marshal(data)
if err != nil {
c.log.Errorf("failed to marshal data: %v", err)
return err
}
var resp *esapiV9.Response
if id == "" {
resp, err = c.cli.Index(
indexName,
bytes.NewReader(dataBytes),
c.cli.Index.WithContext(ctx),
)
} else {
resp, err = c.cli.Create(
indexName, id,
bytes.NewReader(dataBytes),
c.cli.Create.WithContext(ctx),
)
}
if err != nil {
c.log.Errorf("failed to insert document: %v", err)
return err
}
defer func(Body io.ReadCloser) {
if err = Body.Close(); err != nil {
c.log.Errorf("failed to close response body: %v", err)
}
}(resp.Body)
if resp.IsError() {
var errResp *ErrorResponse
if errResp, err = ParseErrorMessage(resp.Body); err != nil {
c.log.Errorf("failed to parse error message: %v", err)
return err
}
c.log.Errorf("insert data failed: %s", errResp.Error.Reason)
return ErrInsertDocument
}
return nil
}
// BatchInsertDocument 批量插入数据
func (c *Client) BatchInsertDocument(ctx context.Context, indexName string, dataSet []interface{}) error {
var buf bytes.Buffer
for _, data := range dataSet {
meta := []byte(`{"index":{}}` + "\n")
dataBytes, err := json.Marshal(data)
if err != nil {
c.log.Errorf("failed to marshal data: %v", err)
continue
}
dataBytes = append(dataBytes, "\n"...)
buf.Grow(len(meta) + len(dataBytes))
buf.Write(meta)
buf.Write(dataBytes)
}
resp, err := c.cli.Bulk(
bytes.NewReader(buf.Bytes()),
c.cli.Bulk.WithContext(ctx),
c.cli.Bulk.WithIndex(indexName),
)
if err != nil {
c.log.Errorf("failed to perform bulk insert: %v", err)
return err
}
defer func(Body io.ReadCloser) {
if err = Body.Close(); err != nil {
c.log.Errorf("failed to close response body: %v", err)
}
}(resp.Body)
if resp.IsError() {
var errResp *ErrorResponse
if errResp, err = ParseErrorMessage(resp.Body); err != nil {
c.log.Errorf("failed to parse error message: %v", err)
return err
}
c.log.Errorf("batch insert data failed: %s", errResp.Error.Reason)
return ErrBatchInsertDocument
}
return nil
}
func (c *Client) UpdateDocument(ctx context.Context, indexName string, pk string, doc interface{}) error {
data, err := json.Marshal(doc)
if err != nil {
c.log.Errorf("failed to marshal data: %v", err)
return err
}
_, err = c.cli.Update(
indexName, pk,
bytes.NewReader(data),
c.cli.Update.WithContext(ctx),
)
if err != nil {
c.log.Errorf("failed to update document: %v", err)
return err
}
return nil
}
// GetDocument 查询数据
func (c *Client) GetDocument(
ctx context.Context,
indexName string,
id string,
sourceFields []string,
out interface{},
) error {
resp, err := c.cli.Get(
indexName, id,
c.cli.Get.WithContext(ctx),
c.cli.Get.WithSource(sourceFields...), // 指定返回的字段
)
if err != nil {
c.log.Errorf("failed to get document: %v", err)
return err
}
if resp.IsError() {
var errResp *ErrorResponse
if errResp, err = ParseErrorMessage(resp.Body); err != nil {
c.log.Errorf("failed to parse error message: %v", err)
return err
}
if resp.StatusCode == 404 {
c.log.Warnf("document not found: %s", errResp.Error.Reason)
return ErrDocumentNotFound
}
c.log.Errorf("get document failed: %s", errResp.Error.Reason)
return ErrGetDocument
}
if err = json.NewDecoder(resp.Body).Decode(&out); err != nil {
c.log.Errorf("failed to decode document: %v", err)
return err
}
return nil
}
func (c *Client) Search(
ctx context.Context,
indexName string,
req *pagination.PagingRequest,
) (*SearchResult, error) {
var query string
ParseQueryString(req.GetQuery())
sortBy := make(map[string]bool)
pageSize := req.GetPageSize()
if pageSize <= 0 {
pageSize = 20 // Default page size
}
return c.search(ctx, indexName, query, nil, sortBy, int(req.GetPage()), int(pageSize))
}
// search 查询数据
//
// @param ctx 上下文
// @param indexName 索引名
// @param query 查询条件，例如：field1:value1 AND field2:value2
// @param sourceFields 指定返回的字段，传入nil表示返回所有字段
// @param sortBy 排序
// @param from 分页的页码
// @param pageSize 分页每页的行数
func (c *Client) search(
ctx context.Context,
indexName string,
query string,
sourceFields []string,
sortBy map[string]bool,
from, pageSize int,
) (*SearchResult, error) {
var sorts []string
for k, v := range sortBy {
if v {
sorts = append(sorts, k+":asc")
} else {
sorts = append(sorts, k+":desc")
}
}
resp, err := c.cli.Search(
c.cli.Search.WithContext(ctx),
c.cli.Search.WithIndex(indexName),
c.cli.Search.WithFrom(from),
c.cli.Search.WithSize(pageSize),
c.cli.Search.WithSort(sorts...),
c.cli.Search.WithQuery(query),
c.cli.Search.WithSource(sourceFields...), // 指定返回的字段
)
if err != nil {
c.log.Errorf("failed to search documents: %v", err)
return nil, err
}
defer func(Body io.ReadCloser) {
if err := Body.Close(); err != nil {
c.log.Errorf("failed to close response body: %v", err)
}
}(resp.Body)
if resp.IsError() {
var errResp *ErrorResponse
if errResp, err = ParseErrorMessage(resp.Body); err != nil {
return nil, err
}
c.log.Errorf("search document failed: %s", errResp.Error.Reason)
return nil, ErrSearchDocument
}
var searchResult SearchResult
if err = json.NewDecoder(resp.Body).Decode(&searchResult); err != nil {
c.log.Errorf("failed to decode search result: %v", err)
return nil, err
}
return &searchResult, nil
}

359
database/elasticsearch/client_test.go Normal file
View File

@@ -0,0 +1,359 @@
package elasticsearch
import (
"context"
"encoding/json"
"fmt"
"math/rand"
"strconv"
"strings"
"testing"
"time"
"github.com/go-kratos/kratos/v2/log"
"github.com/stretchr/testify/assert"
conf "github.com/tx7do/kratos-bootstrap/api/gen/go/conf/v1"
)
const (
userIndex = "user"
tweetIndex = "tweet"
sensorIndex = "sensor"
)
type User struct {
Name string `json:"name"`
Age int `json:"age"`
Phone string `json:"phone"`
Birth time.Time `json:"birth"`
Height float32 `json:"height"`
Smoke bool `json:"smoke"`
Home string `json:"home"`
}
// UserMapping 定义用户mapping
const UserMapping = `
{
"mappings": {
"properties": {
"name": {"type": "text"},
"age": {"type": "byte"},
"phone": {"type": "text"},
"birth": {"type": "date"},
"height": {"type": "float"},
"smoke": {"type": "boolean"},
"home": {"type": "geo_point"}
}
}
}`
type Tweet struct {
User string `json:"user"` // 用户
Message string `json:"message"` // 微博内容
Retweets int `json:"retweets"` // 转发数
Image string `json:"image,omitempty"` // 图片
Created time.Time `json:"created,omitempty"` // 创建时间
Tags []string `json:"tags,omitempty"` // 标签
Location string `json:"location,omitempty"` //位置
//Suggest *elasticsearchV9.SuggestField `json:"suggest_field,omitempty"`
}
const TweetMapping = `
{
"mappings": {
"properties": {
"user": {"type": "keyword"},
"message": {"type": "text"},
"image": {"type": "keyword"},
"created": {"type": "date"},
"tags": {"type": "keyword"},
"location": {"type": "geo_point"},
"suggest_field": {"type": "completion"}
}
}
}`
type Sensor struct {
Id int `json:"id" bson:"_id,omitempty"`
Type string `json:"type" bson:"type,omitempty"`
Location string `json:"location,omitempty" bson:"location,omitempty"`
}
type SensorData struct {
Id string `json:"id" bson:"_id,omitempty"`
Time time.Time `json:"time" bson:"created,omitempty"`
SensorId int `json:"sensor_id" bson:"sensor_id,omitempty"`
Temperature float64 `json:"temperature" bson:"temperature,omitempty"`
CPU float64 `json:"cpu" bson:"cpu,omitempty"`
}
const SensorMapping = `
{
"mappings": {
"properties": {
"sensor_id": {"type": "integer"},
"temperature": {"type": "double"},
"cpu": {"type": "double"},
"location": {"type": "geo_point"}
}
}
}`
func createTestClient() *Client {
cli, _ := NewClient(
log.DefaultLogger,
&conf.Bootstrap{
Data: &conf.Data{
ElasticSearch: &conf.Data_ElasticSearch{
Addresses: []string{"http://localhost:9200"},
Username: "elastic",
Password: "elastic",
},
},
},
)
return cli
}
func TestNewClient(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
client.CheckConnectStatus()
}
func TestCreateIndex(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
var esCtx = context.Background()
{
_ = client.DeleteIndex(esCtx, userIndex)
err := client.CreateIndex(esCtx, userIndex, UserMapping, "")
assert.Nil(t, err)
}
{
_ = client.DeleteIndex(esCtx, tweetIndex)
err := client.CreateIndex(esCtx, tweetIndex, TweetMapping, "")
assert.Nil(t, err)
}
{
_ = client.DeleteIndex(esCtx, sensorIndex)
err := client.CreateIndex(esCtx, sensorIndex, SensorMapping, "")
assert.Nil(t, err)
}
}
func TestDeleteIndex(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
var esCtx = context.Background()
err := client.DeleteIndex(esCtx, userIndex)
assert.Nil(t, err)
err = client.DeleteIndex(esCtx, tweetIndex)
assert.Nil(t, err)
err = client.DeleteIndex(esCtx, sensorIndex)
assert.Nil(t, err)
}
func TestInsertDocument(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
var esCtx = context.Background()
{
// http://localhost:9200/user/_search?q=*&pretty
loc, _ := time.LoadLocation("Local")
birth, _ := time.ParseInLocation("2006-01-02", "1991-04-25", loc)
userOne := User{
Name: "张三",
Age: 23,
Phone: "17600000000",
Birth: birth,
Height: 170.5,
Home: "41.40338,2.17403",
}
err := client.InsertDocument(esCtx, userIndex, "", userOne)
assert.Nil(t, err)
}
{
tweetOne := Tweet{User: "olive", Message: "打酱油的一天", Retweets: 0}
err := client.InsertDocument(esCtx, tweetIndex, "", tweetOne)
assert.Nil(t, err)
}
}
func TestBatchInsertDocument(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
var esCtx = context.Background()
{
loc, _ := time.LoadLocation("Local")
// 生日
birthSlice := []string{"1991-04-25", "1990-01-15", "1989-11-05", "1988-01-25", "1994-10-12"}
// 姓名
nameSlice := []string{"李四", "张飞", "赵云", "关羽", "刘备"}
var users []interface{}
for i := 1; i < 20; i++ {
birth, _ := time.ParseInLocation("2006-01-02", birthSlice[rand.Intn(len(birthSlice))], loc)
height, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", rand.Float32()+175.0), 32)
user := User{
Name: nameSlice[rand.Intn(len(nameSlice))],
Age: rand.Intn(10) + 18,
Phone: "1760000000" + strconv.Itoa(i),
Birth: birth,
Height: float32(height),
Home: "41.40338,2.17403",
}
users = append(users, user)
}
err := client.BatchInsertDocument(esCtx, userIndex, users)
assert.Nil(t, err)
}
}
func TestGetDocument(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
var esCtx = context.Background()
var user User
const id = "N_1fm5cBE8GqVkmNBLNY"
err := client.GetDocument(esCtx, userIndex, id, nil, &user)
assert.Equal(t, err, ErrDocumentNotFound)
assert.NotNil(t, user)
}
func TestSearch(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
var esCtx = context.Background()
//// 创建索引并插入测试数据
//_ = client.DeleteIndex(esCtx, userIndex)
//err := client.CreateIndex(esCtx, userIndex, UserMapping, "")
//assert.Nil(t, err)
//
//userOne := User{
// Name: "张三",
// Age: 23,
// Phone: "17600000000",
// Height: 170.5,
// Home: "41.40338,2.17403",
//}
//err = client.InsertDocument(esCtx, userIndex, "", userOne)
//assert.Nil(t, err)
// 测试Search方法
query := "name:张三"
sortBy := map[string]bool{"age": true}
from := 0
pageSize := 10
searchResult, err := client.search(
esCtx, userIndex, query, nil, sortBy, from, pageSize,
)
assert.Nil(t, err)
assert.NotNil(t, searchResult)
var users []User
for _, hit := range searchResult.Hits.Hits {
var user User
if err = json.Unmarshal(hit.Source, &user); err != nil {
t.Errorf("Failed to unmarshal hit: %v", err)
continue
}
users = append(users, user)
}
t.Logf("Search result: %v", users)
}
func TestMergeOptions(t *testing.T) {
mapping := `{
"properties": {
"name": {
"type": "text"
},
"age": {
"type": "integer"
}
}
}`
settings := `{
"index": {
"number_of_shards": 1,
"number_of_replicas": 0
}
}`
//expected := `{"mappings":{"properties":{"name":{"type":"text"},"age":{"type":"integer"}}},"settings":{"index":{"number_of_shards":1,"number_of_replicas":0}}}`
result, err := MergeOptions(mapping, settings)
assert.Nil(t, err)
//assert.Equal(t, expected, result)
t.Log(result)
}
func TestParseQueryString(t *testing.T) {
// 测试单个键值对的查询字符串
query := `{"name":"张三"}`
result := ParseQueryString(query)
assert.NotNil(t, result)
assert.Equal(t, []string{"name:张三"}, result)
// 测试多个键值对的查询字符串
query = `[{"name":"张三"},{"age":"23"}]`
result = ParseQueryString(query)
assert.NotNil(t, result)
assert.Equal(t, []string{"name:张三", "age:23"}, result)
t.Log(strings.Join(result, " AND "))
// 测试无效的查询字符串
query = `invalid`
result = ParseQueryString(query)
assert.Nil(t, result)
}
func TestMakeQueryString(t *testing.T) {
// 测试 AND 查询
andQuery := `{"name":"张三","age":"23"}`
orQuery := ``
result := MakeQueryString(andQuery, orQuery)
assert.Equal(t, "name:张三 AND age:23", result)
// 测试 OR 查询
andQuery = ``
orQuery = `[{"city":"北京"},{"country":"中国"}]`
result = MakeQueryString(andQuery, orQuery)
assert.Equal(t, "city:北京 OR country:中国", result)
// 测试 AND 和 OR 查询同时存在
andQuery = `{"name":"张三"}`
orQuery = `[{"city":"北京"},{"country":"中国"}]`
result = MakeQueryString(andQuery, orQuery)
assert.Equal(t, "name:张三 AND (city:北京 OR country:中国)", result)
// 测试空查询
andQuery = ``
orQuery = ``
result = MakeQueryString(andQuery, orQuery)
assert.Equal(t, "", result)
}

38
database/elasticsearch/errors.go Normal file
View File

@@ -0,0 +1,38 @@
package elasticsearch
import "github.com/go-kratos/kratos/v2/errors"
var (
// ErrRequestFailed is returned when a request to Elasticsearch fails.
ErrRequestFailed = errors.InternalServer("REQUEST_FAILED", "request failed")
// ErrIndexNotFound is returned when the specified index does not exist.
ErrIndexNotFound = errors.InternalServer("INDEX_NOT_FOUND", "index not found")
// ErrIndexAlreadyExists is returned when trying to create an index that already exists.
ErrIndexAlreadyExists = errors.InternalServer("INDEX_ALREADY_EXISTS", "index already exists")
ErrCreateIndex = errors.InternalServer("CREATE_INDEX_FAILED", "failed to create index")
ErrDeleteIndex = errors.InternalServer("DELETE_INDEX_FAILED", "failed to delete index")
// ErrDocumentNotFound is returned when a document is not found in the index.
ErrDocumentNotFound = errors.InternalServer("DOCUMENT_NOT_FOUND", "document not found")
// ErrDocumentAlreadyExists is returned when trying to create a document that already exists.
ErrDocumentAlreadyExists = errors.InternalServer("DOCUMENT_ALREADY_EXISTS", "document already exists")
// ErrInvalidQuery is returned when the query provided to Elasticsearch is invalid.
ErrInvalidQuery = errors.InternalServer("INVALID_QUERY", "invalid query")
// ErrUnmarshalResponse is returned when the response from Elasticsearch cannot be unmarshalled.
ErrUnmarshalResponse = errors.InternalServer("UNMARSHAL_RESPONSE_FAILED", "failed to unmarshal response")
ErrInsertDocument = errors.InternalServer("INSERT_DOCUMENT_FAILED", "failed to insert document")
ErrBatchInsertDocument = errors.InternalServer("BATCH_INSERT_DOCUMENT_FAILED", "failed to batch insert documents")
ErrGetDocument = errors.InternalServer("GET_DOCUMENT_FAILED", "failed to get document")
ErrSearchDocument = errors.InternalServer("SEARCH_DOCUMENT_FAILED", "failed to search document")
)

35
database/elasticsearch/go.mod Normal file
View File

@@ -0,0 +1,35 @@
module github.com/tx7do/kratos-bootstrap/database/elasticsearch
go 1.23.0
toolchain go1.23.3
replace github.com/tx7do/kratos-bootstrap/api => ../../api
require (
github.com/elastic/go-elasticsearch/v9 v9.0.0
github.com/go-kratos/kratos/v2 v2.8.4
github.com/stretchr/testify v1.10.0
github.com/tx7do/kratos-bootstrap/api v0.0.25
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/elastic/elastic-transport-go/v8 v8.7.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/gnostic v0.7.0 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sys v0.33.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/grpc v1.73.0 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

1565
database/elasticsearch/go.sum Normal file
View File

File diff suppressed because it is too large Load Diff

38
database/elasticsearch/types.go Normal file
View File

@@ -0,0 +1,38 @@
package elasticsearch
import "encoding/json"
// ErrorResponse 表示 Elasticsearch 错误响应的结构
type ErrorResponse struct {
Error struct {
RootCause []struct {
Type string `json:"type"`
Reason string `json:"reason"`
} `json:"root_cause"`
Type string `json:"type"`
Reason string `json:"reason"`
CausedBy struct {
Type string `json:"type"`
Reason string `json:"reason"`
} `json:"caused_by,omitempty"`
} `json:"error"`
Status int `json:"status"`
}
type SearchResult struct {
Took int `json:"took"`
TimedOut bool `json:"timed_out"`
Hits struct {
Total struct {
Value int `json:"value"`
Relation string `json:"relation"`
} `json:"total"`
Hits []struct {
Index string `json:"_index"`
Type string `json:"_type"`
ID string `json:"_id"`
Score float64 `json:"_score"`
Source json.RawMessage `json:"_source"`
} `json:"hits"`
} `json:"hits"`
}

111
database/elasticsearch/utils.go Normal file
View File

@@ -0,0 +1,111 @@
package elasticsearch
import (
"encoding/json"
"io"
"strings"
"github.com/go-kratos/kratos/v2/encoding"
_ "github.com/go-kratos/kratos/v2/encoding/json"
"github.com/go-kratos/kratos/v2/log"
)
// ParseErrorMessage 解析 Elasticsearch 错误消息
func ParseErrorMessage(body io.ReadCloser) (*ErrorResponse, error) {
defer body.Close()
var errorResponse ErrorResponse
if err := json.NewDecoder(body).Decode(&errorResponse); err != nil {
return nil, ErrUnmarshalResponse
}
return &errorResponse, nil
}
// MergeOptions 合并 Elasticsearch 索引的映射和设置
func MergeOptions(mapping, settings string) (string, error) {
codec := encoding.GetCodec("json")
body := make(map[string]interface{})
if mapping != "" {
var mappingObj map[string]interface{}
if err := codec.Unmarshal([]byte(mapping), &mappingObj); err != nil {
log.Errorf("failed to unmarshal mapping: %v", err)
return "", err
}
if existingMappings, ok := mappingObj["mappings"]; ok {
body["mappings"] = existingMappings
} else {
body["mappings"] = mappingObj
}
}
if settings != "" {
var settingsObj map[string]interface{}
if err := codec.Unmarshal([]byte(settings), &settingsObj); err != nil {
log.Errorf("failed to unmarshal settings: %v", err)
return "", err
}
// 检查 settings 是否包含 settings 字段
if existingSettings, ok := settingsObj["settings"]; ok {
body["settings"] = existingSettings
} else {
body["settings"] = settingsObj
}
}
bodyBytes, err := codec.Marshal(body)
if err != nil {
log.Errorf("failed to marshal request body: %v", err)
return "", err
}
return string(bodyBytes), nil
}
func ParseQueryString(query string) []string {
codec := encoding.GetCodec("json")
var err error
queryMap := make(map[string]string)
if err = codec.Unmarshal([]byte(query), &queryMap); err == nil {
var queries []string
for k, v := range queryMap {
queries = append(queries, k+":"+v)
}
return queries
}
var queryMapArray []map[string]string
if err = codec.Unmarshal([]byte(query), &queryMapArray); err == nil {
var queries []string
for _, item := range queryMapArray {
for k, v := range item {
queries = append(queries, k+":"+v)
}
}
return queries
}
return nil
}
func MakeQueryString(andQuery, orQuery string) string {
a := ParseQueryString(andQuery)
o := ParseQueryString(orQuery)
if len(a) == 0 && len(o) == 0 {
return ""
}
if len(a) > 0 && len(o) == 0 {
return strings.Join(a, " AND ")
} else if len(a) == 0 && len(o) > 0 {
return strings.Join(o, " OR ")
} else if len(a) > 0 && len(o) > 0 {
return strings.Join(a, " AND ") + " AND (" + strings.Join(o, " OR ") + ")"
} else {
return strings.Join(a, " AND ") + " AND " + strings.Join(o, " OR ")
}
}

56
database/influxdb/README.md Normal file
View File

@@ -0,0 +1,56 @@
# InfluxDB
### Docker部署
pull image
```bash
docker pull bitnami/influxdb:latest
```
#### 2.x
```bash
docker run -itd \
--name influxdb2-server \
-p 8086:8086 \
-e INFLUXDB_HTTP_AUTH_ENABLED=true \
-e INFLUXDB_ADMIN_USER=admin \
-e INFLUXDB_ADMIN_USER_PASSWORD=123456789 \
-e INFLUXDB_ADMIN_USER_TOKEN=admintoken123 \
-e INFLUXDB_DB=my_database \
bitnami/influxdb:2.7.11
```
create admin user sql script:
```sql
create user "admin" with password '123456789' with all privileges
```
管理后台: <http://localhost:8086/>
#### 3.x
```bash
docker run -itd \
--name influxdb3-server \
-p 8181:8181 \
-e INFLUXDB_NODE_ID=0 \
-e INFLUXDB_HTTP_PORT_NUMBER=8181 \
-e INFLUXDB_HTTP_AUTH_ENABLED=true \
-e INFLUXDB_CREATE_ADMIN_TOKEN=yes \
-e INFLUXDB_DB=my_database \
bitnami/influxdb:latest
docker run -itd \
--name influxdb3-explorer \
-p 8888:80 \
-p 8889:8888 \
quay.io/influxdb/influxdb3-explorer:latest \
--mode=admin
```
这个版本分离出来一个管理后台 InfluxDB Explorer：<http://localhost:8888/>
在管理后台填写：`http://host.docker.internal:8181`

115
database/influxdb/client.go
View File

@@ -1,29 +1,124 @@
package influxdb
import (
"github.com/InfluxCommunity/influxdb3-go/influxdb3"
"context"
"github.com/go-kratos/kratos/v2/encoding"
_ "github.com/go-kratos/kratos/v2/encoding/json"
"github.com/go-kratos/kratos/v2/log"
"github.com/InfluxCommunity/influxdb3-go/v2/influxdb3"
conf "github.com/tx7do/kratos-bootstrap/api/gen/go/conf/v1"
)
func NewInfluxClient(cfg *conf.Bootstrap, l *log.Helper) *influxdb3.Client {
type Client struct {
cli *influxdb3.Client
log *log.Helper
codec encoding.Codec
}
func NewClient(logger log.Logger, cfg *conf.Bootstrap) (*Client, error) {
c := &Client{
log: log.NewHelper(log.With(logger, "module", "influxdb-client")),
codec: encoding.GetCodec("json"),
}
if err := c.createInfluxdbClient(cfg); err != nil {
return nil, err
}
return c, nil
}
// createInfluxdbClient 创建InfluxDB客户端
func (c *Client) createInfluxdbClient(cfg *conf.Bootstrap) error {
if cfg.Data == nil || cfg.Data.Influxdb == nil {
l.Warn("influxdb config is nil")
return nil
}
client, err := influxdb3.New(influxdb3.ClientConfig{
Host: cfg.Data.Influxdb.Address,
Token: cfg.Data.Influxdb.Token,
Database: cfg.Data.Influxdb.Bucket,
Organization: cfg.Data.Influxdb.Organization,
Host: cfg.Data.Influxdb.GetHost(),
Token: cfg.Data.Influxdb.GetToken(),
Database: cfg.Data.Influxdb.GetDatabase(),
Organization: cfg.Data.Influxdb.GetOrganization(),
})
if err != nil {
l.Fatalf("failed opening connection to influxdb: %v", err)
return nil
c.log.Errorf("failed to create influxdb client: %v", err)
return err
}
return client
c.cli = client
return nil
}
// Close 关闭InfluxDB客户端
func (c *Client) Close() {
if c.cli == nil {
c.log.Warn("influxdb client is nil, nothing to close")
return
}
if err := c.cli.Close(); err != nil {
c.log.Errorf("failed to close influxdb client: %v", err)
} else {
c.log.Info("influxdb client closed successfully")
}
}
// Query 查询数据
func (c *Client) Query(ctx context.Context, query string) (*influxdb3.QueryIterator, error) {
if c.cli == nil {
return nil, ErrInfluxDBClientNotInitialized
}
result, err := c.cli.Query(
ctx,
query,
influxdb3.WithQueryType(influxdb3.InfluxQL),
)
if err != nil {
c.log.Errorf("failed to query data: %v", err)
return nil, ErrInfluxDBQueryFailed
}
return result, nil
}
// Insert 插入数据
func (c *Client) Insert(ctx context.Context, point *influxdb3.Point) error {
if c.cli == nil {
return ErrInfluxDBClientNotInitialized
}
if point == nil {
return ErrInvalidPoint
}
points := []*influxdb3.Point{point}
if err := c.cli.WritePoints(ctx, points); err != nil {
c.log.Errorf("failed to insert data: %v", err)
return ErrInsertFailed
}
return nil
}
// BatchInsert 批量插入数据
func (c *Client) BatchInsert(ctx context.Context, points []*influxdb3.Point) error {
if c.cli == nil {
return ErrInfluxDBClientNotInitialized
}
if len(points) == 0 {
return ErrNoPointsToInsert
}
if err := c.cli.WritePoints(ctx, points); err != nil {
c.log.Errorf("failed to batch insert data: %v", err)
return ErrBatchInsertFailed
}
return nil
}

211
database/influxdb/client_test.go Normal file
View File

@@ -0,0 +1,211 @@
package influxdb
import (
"context"
"testing"
"time"
"github.com/InfluxCommunity/influxdb3-go/v2/influxdb3"
"github.com/go-kratos/kratos/v2/log"
"github.com/stretchr/testify/assert"
"github.com/tx7do/go-utils/trans"
conf "github.com/tx7do/kratos-bootstrap/api/gen/go/conf/v1"
"google.golang.org/protobuf/types/known/timestamppb"
)
type Candle struct {
Symbol *string
Open *float64
High *float64
Low *float64
Close *float64
Volume *float64
StartTime *timestamppb.Timestamp
}
func (c *Candle) GetSymbol() string {
if c.Symbol != nil {
return *c.Symbol
}
return ""
}
func (c *Candle) GetOpen() float64 {
if c.Open != nil {
return *c.Open
}
return 0.0
}
func (c *Candle) GetHigh() float64 {
if c.High != nil {
return *c.High
}
return 0.0
}
func (c *Candle) GetLow() float64 {
if c.Low != nil {
return *c.Low
}
return 0.0
}
func (c *Candle) GetClose() float64 {
if c.Close != nil {
return *c.Close
}
return 0.0
}
func (c *Candle) GetVolume() float64 {
if c.Volume != nil {
return *c.Volume
}
return 0.0
}
func (c *Candle) GetStartTime() *timestamppb.Timestamp {
if c.StartTime != nil {
return c.StartTime
}
return timestamppb.Now()
}
type CandleMapper struct{}
var candleMapper CandleMapper
func (m *CandleMapper) ToPoint(data *Candle) *influxdb3.Point {
p := influxdb3.NewPoint(
"candles",
map[string]string{"s": data.GetSymbol()},
nil,
data.StartTime.AsTime(),
)
p.
SetDoubleField("o", data.GetOpen()).
SetDoubleField("h", data.GetHigh()).
SetDoubleField("l", data.GetLow()).
SetDoubleField("c", data.GetClose()).
SetDoubleField("v", data.GetVolume())
return p
}
func (m *CandleMapper) ToData(point *influxdb3.Point) *Candle {
symbol, _ := point.GetTag("s")
return &Candle{
Symbol: &symbol,
Open: point.GetDoubleField("o"),
High: point.GetDoubleField("h"),
Low: point.GetDoubleField("l"),
Close: point.GetDoubleField("c"),
Volume: point.GetDoubleField("v"),
StartTime: timestamppb.New(point.Values.Timestamp),
}
}
func createTestClient() *Client {
cli, _ := NewClient(
log.DefaultLogger,
&conf.Bootstrap{
Data: &conf.Data{
Influxdb: &conf.Data_InfluxDB{
Host: "http://localhost:8181",
Token: "apiv3_yYde4mJo0BYC7Ipi_00ZEex-A8if4swdqTBXiO-lCUDKhsIavHlRCQfo3p_DzI7S34ADHOC7Qxf600VVgW6LQQ",
Database: "finances",
Organization: "primary",
},
},
},
)
return cli
}
func TestNewClient(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
}
func TestClient_Insert(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
item := &Candle{
StartTime: timestamppb.New(time.Now()),
Symbol: trans.Ptr("AAPL"),
Open: trans.Ptr(1.0),
High: trans.Ptr(2.0),
Low: trans.Ptr(3.0),
Close: trans.Ptr(4.0),
Volume: trans.Ptr(1000.0),
}
point := candleMapper.ToPoint(item)
err := client.Insert(context.Background(), point)
assert.NoError(t, err)
}
func TestClient_BatchInsert(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
items := []*Candle{
{
StartTime: timestamppb.New(time.Now()),
Symbol: trans.Ptr("AAPL"),
Open: trans.Ptr(1.0),
High: trans.Ptr(2.0),
Low: trans.Ptr(3.0),
Close: trans.Ptr(4.0),
Volume: trans.Ptr(1000.0),
},
}
var points []*influxdb3.Point
for _, item := range items {
point := candleMapper.ToPoint(item)
points = append(points, point)
}
err := client.BatchInsert(
context.Background(),
points,
)
assert.NoError(t, err)
}
func TestClient_Query(t *testing.T) {
client := createTestClient()
assert.NotNil(t, client)
ctx := context.Background()
sql := `SELECT * FROM candles`
iterator, err := client.Query(ctx, sql)
assert.NoError(t, err)
for iterator.Next() {
point, _ := iterator.AsPoints().AsPoint()
candle := candleMapper.ToData(point)
t.Logf("[%v] Candle: %s, Open: %f, High: %f, Low: %f, Close: %f, Volume: %f\n",
candle.GetStartTime().AsTime().String(),
candle.GetSymbol(),
candle.GetOpen(), candle.GetHigh(), candle.GetLow(), candle.GetClose(), candle.GetVolume(),
)
}
candles, err := Query(ctx, client, sql, &candleMapper)
assert.NoError(t, err)
for _, candle := range candles {
t.Logf("Candle: %s, Open: %f, High: %f, Low: %f, Close: %f, Volume: %f\n",
candle.GetSymbol(),
candle.GetOpen(), candle.GetHigh(), candle.GetLow(), candle.GetClose(), candle.GetVolume(),
)
}
}

25
database/influxdb/errors.go Normal file
View File

@@ -0,0 +1,25 @@
package influxdb
import "github.com/go-kratos/kratos/v2/errors"
var (
ErrInfluxDBClientNotInitialized = errors.InternalServer("INFLUXDB_CLIENT_NOT_INITIALIZED", "client not initialized")
ErrInfluxDBConnectFailed = errors.InternalServer("INFLUXDB_CONNECT_FAILED", "connect failed")
ErrInfluxDBCreateDatabaseFailed = errors.InternalServer("INFLUXDB_CREATE_DATABASE_FAILED", "database create failed")
ErrInfluxDBQueryFailed = errors.InternalServer("INFLUXDB_QUERY_FAILED", "query failed")
ErrClientNotConnected = errors.InternalServer("INFLUXDB_CLIENT_NOT_CONNECTED", "client not connected")
ErrInvalidPoint = errors.InternalServer("INFLUXDB_INVALID_POINT", "invalid point")
ErrNoPointsToInsert = errors.InternalServer("INFLUXDB_NO_POINTS_TO_INSERT", "no points to insert")
ErrEmptyData = errors.InternalServer("INFLUXDB_EMPTY_DATA", "empty data")
ErrBatchInsertFailed = errors.InternalServer("INFLUXDB_BATCH_INSERT_FAILED", "batch insert failed")
ErrInsertFailed = errors.InternalServer("INFLUXDB_INSERT_FAILED", "insert failed")
)

36
database/influxdb/go.mod
View File

@@ -1,36 +1,42 @@
module github.com/tx7do/kratos-bootstrap/database/influxdb
go 1.23.0
go 1.23.10
toolchain go1.23.3
toolchain go1.24.4
replace github.com/tx7do/kratos-bootstrap/api => ../../api
require (
github.com/InfluxCommunity/influxdb3-go v0.14.0
github.com/InfluxCommunity/influxdb3-go/v2 v2.8.0
github.com/go-kratos/kratos/v2 v2.8.4
github.com/tx7do/kratos-bootstrap/api v0.0.21
github.com/stretchr/testify v1.10.0
github.com/tx7do/go-utils v1.1.29
github.com/tx7do/kratos-bootstrap/api v0.0.25
google.golang.org/protobuf v1.36.6
)
require (
github.com/apache/arrow/go/v15 v15.0.2 // indirect
github.com/apache/arrow-go/v18 v18.3.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/frankban/quicktest v1.14.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/influxdata/line-protocol/v2 v2.2.1 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b // indirect
golang.org/x/mod v0.24.0 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/sync v0.15.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.25.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/tools v0.34.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
google.golang.org/grpc v1.72.2 // indirect
google.golang.org/protobuf v1.36.6 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/grpc v1.73.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

92
database/influxdb/go.sum
View File

@@ -1,7 +1,11 @@
github.com/InfluxCommunity/influxdb3-go v0.14.0 h1:lFEJRZM+hQzuCz36k6YKeE6CE9oKBfKkIO2bYLghar0=
github.com/InfluxCommunity/influxdb3-go v0.14.0/go.mod h1:+0XXsEt0XMP0o7WBvDXLCJ8MxmxfgjG3mLrYPRRWsLs=
github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE=
github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA=
github.com/InfluxCommunity/influxdb3-go/v2 v2.8.0 h1:auHy7TmHQJVRs+r59k+UIlN9yuY4eFq7d6xrsGSo0E8=
github.com/InfluxCommunity/influxdb3-go/v2 v2.8.0/go.mod h1:wccnTQV9OQ9XvW7ttXINSccyzSmaADzYFheoCHW2sCs=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/apache/arrow-go/v18 v18.3.1 h1:oYZT8FqONiK74JhlH3WKVv+2NKYoyZ7C2ioD4Dj3ixk=
github.com/apache/arrow-go/v18 v18.3.1/go.mod h1:12QBya5JZT6PnBihi5NJTzbACrDGXYkrgjujz3MRQXU=
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -20,13 +24,15 @@ github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98=
@@ -36,10 +42,12 @@ github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod
github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY=
github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE=
github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM=
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -49,6 +57,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -57,52 +69,56 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tx7do/go-utils v1.1.29 h1:kO1JDMVX++ZY4+aXGk3pOtDz5WBPDA3LxhIWkzXkvH8=
github.com/tx7do/go-utils v1.1.29/go.mod h1:bmt7c85QmHURtd7h6QOu7k0QKOJTwjJ+cFP29nljdSw=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b h1:QoALfVG9rhQ/M7vYDScfPdWjGL9dlsVVM5VGh7aKoAA=
golang.org/x/exp v0.0.0-20250531010427-b6e5de432a8b/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY=
go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o=
gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8=
google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

96
database/influxdb/mapper.go Normal file
View File

@@ -0,0 +1,96 @@
package influxdb
import (
"context"
"github.com/InfluxCommunity/influxdb3-go/v2/influxdb3"
)
// Mapper 数据转换的接口
type Mapper[T any] interface {
// ToPoint 将数据转换为InfluxDB的Point格式
ToPoint(data *T) *influxdb3.Point
// ToData 将InfluxDB的Point转换为原始数据
ToData(point *influxdb3.Point) *T
}
// Insert 插入数据
func Insert[T any](ctx context.Context, c *Client, data *T, mapper Mapper[T]) error {
if c.cli == nil {
return ErrClientNotConnected
}
if data == nil {
return ErrEmptyData
}
point := mapper.ToPoint(data)
if point == nil {
return ErrInvalidPoint
}
err := c.Insert(ctx, point)
if err != nil {
return err
}
return nil
}
// BatchInsert 批量插入数据
func BatchInsert[T any](ctx context.Context, c *Client, data []*T, mapper Mapper[T]) error {
if c.cli == nil {
return ErrClientNotConnected
}
if len(data) == 0 {
return ErrEmptyData
}
points := make([]*influxdb3.Point, len(data))
for i, d := range data {
point := mapper.ToPoint(d)
if point == nil {
return ErrInvalidPoint
}
points[i] = point
}
err := c.BatchInsert(ctx, points)
if err != nil {
return err
}
return nil
}
// Query 查询数据
func Query[T any](ctx context.Context, c *Client, query string, mapper Mapper[T]) ([]*T, error) {
if c.cli == nil {
return nil, ErrClientNotConnected
}
iterator, err := c.Query(ctx, query)
if err != nil {
return nil, err
}
var dataset []*T
for iterator.Next() {
point, _ := iterator.AsPoints().AsPoint()
if point == nil {
return nil, ErrInvalidPoint
}
data := mapper.ToData(point)
dataset = append(dataset, data)
}
if iterator.Err() != nil {
return nil, iterator.Err()
}
return dataset, nil
}

3
tag.bat
View File

@@ -11,9 +11,10 @@ git tag tracer/v0.0.10 --force
git tag database/ent/v0.0.10 --force
git tag database/gorm/v0.0.10 --force
git tag database/mongodb/v0.0.10 --force
git tag database/influxdb/v0.0.10 --force
git tag database/influxdb/v0.0.11 --force
git tag database/cassandra/v0.0.10 --force
git tag database/clickhouse/v0.0.10 --force
git tag database/elasticsearch/v0.0.1 --force
git tag registry/v0.1.0 --force
git tag registry/consul/v0.1.0 --force