diff --git a/agent.go b/agent.go index fd4c7cf..ac4e4b5 100644 --- a/agent.go +++ b/agent.go @@ -11,6 +11,7 @@ import ( "github.com/fumiama/deepinfra/model" "github.com/pkg/errors" zero "github.com/wdvxdr1123/ZeroBot" + "github.com/wdvxdr1123/ZeroBot/message" ) var ( @@ -58,6 +59,42 @@ func (ag *Agent) AddRequest(grp int64, req *zero.APIRequest) { ag.log.Add(grp, req, true) } +// SetViewImageAPI 为 agent 增加识图功能, 需要模型支持视觉 +func (ag *Agent) SetViewImageAPI(api deepinfra.API, p model.Protocol) { + ag.log.SetPreModelize(func(s *fmt.Stringer) { + o := *s + if ev, ok := o.(*Event); ok { + hasset := false + msgs := message.ParseMessage(ev.Message) + for i, msg := range msgs { + if msg.Type != "image" { + continue + } + if _, ok := msg.Data["__agent_desc__"]; ok { + continue + } + u := msg.Data["url"] + if !strings.HasPrefix(u, "http") { + continue + } + m := p.User(model.NewContentImageURL(u), model.NewContentText("使用简洁清晰明确的一段中文纯文本描述图片")) + desc, err := api.Request(m) + if err != nil { + continue + } + msgs[i].Data["__agent_desc__"] = desc + hasset = true + } + if hasset { + raw, err := json.Marshal(&msgs) + if err == nil { + ev.Message = raw + } + } + } + }) +} + // GetAction get OneBot CallAction from LLM and add it to context. // // Note: diff --git a/go.mod b/go.mod index d8b521c..f4bbfaa 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7 - github.com/fumiama/deepinfra v0.0.0-20250920170049-e3d1b92cc3a1 + github.com/fumiama/deepinfra v0.0.0-20250924134153-348633b5c6fb github.com/pkg/errors v0.9.1 github.com/wdvxdr1123/ZeroBot v1.8.2-0.20250921063512-13752a73d444 gopkg.in/yaml.v3 v3.0.1 @@ -12,9 +12,11 @@ require ( require ( github.com/FloatTech/ttl v0.0.0-20240716161252-965925764562 // indirect + github.com/fumiama/imgsz v0.0.4 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/tidwall/gjson v1.17.3 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + golang.org/x/image v0.16.0 // indirect golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect ) diff --git a/go.sum b/go.sum index 327dd5c..d899243 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,10 @@ github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7/go.mod h1:vD7Ra3Q9o github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fumiama/deepinfra v0.0.0-20250920170049-e3d1b92cc3a1 h1:6PglFpNVm3DalGyRldacW2/v4jGWwn3v3q1tr2PhbVQ= -github.com/fumiama/deepinfra v0.0.0-20250920170049-e3d1b92cc3a1/go.mod h1:wW05PQSn8mo1mZIoa6LBUE+3xIBjkoONvnfPTV5ZOhY= +github.com/fumiama/deepinfra v0.0.0-20250924134153-348633b5c6fb h1:d0gl31nyOER4O3K0wpepont8flrDtzZsKACeWGxARX0= +github.com/fumiama/deepinfra v0.0.0-20250924134153-348633b5c6fb/go.mod h1:HgRu6n9XgeyNki7wHHlTa75ORiv3UlYZafqOAoazhOI= +github.com/fumiama/imgsz v0.0.4 h1:Lsasu2hdSSFS+vnD+nvR1UkiRMK7hcpyYCC0FzgSMFI= +github.com/fumiama/imgsz v0.0.4/go.mod h1:bISOQVTlw9sRytPwe8ir7tAaEmyz9hSNj9n8mXMBG0E= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -24,6 +26,8 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/wdvxdr1123/ZeroBot v1.8.2-0.20250921063512-13752a73d444 h1:7aYFXzvVr2zuxBvqrGaJb24Z4W12aXBdW8DuE1mteE4= github.com/wdvxdr1123/ZeroBot v1.8.2-0.20250921063512-13752a73d444/go.mod h1:C86nQ0gIdAri4K2vg8IIQIslt08zzrKMcqYt8zhkx1M= +golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw= +golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=