feat(chat): agent add api SetViewImageAPI

agent.go

@@ -11,6 +11,7 @@ import (
 	"github.com/fumiama/deepinfra/model"
 	"github.com/pkg/errors"
 	zero "github.com/wdvxdr1123/ZeroBot"
+	"github.com/wdvxdr1123/ZeroBot/message"
 )
 
 var (
@@ -58,6 +59,42 @@ func (ag *Agent) AddRequest(grp int64, req *zero.APIRequest) {
 	ag.log.Add(grp, req, true)
 }
 
+// SetViewImageAPI 为 agent 增加识图功能, 需要模型支持视觉
+func (ag *Agent) SetViewImageAPI(api deepinfra.API, p model.Protocol) {
+	ag.log.SetPreModelize(func(s *fmt.Stringer) {
+		o := *s
+		if ev, ok := o.(*Event); ok {
+			hasset := false
+			msgs := message.ParseMessage(ev.Message)
+			for i, msg := range msgs {
+				if msg.Type != "image" {
+					continue
+				}
+				if _, ok := msg.Data["__agent_desc__"]; ok {
+					continue
+				}
+				u := msg.Data["url"]
+				if !strings.HasPrefix(u, "http") {
+					continue
+				}
+				m := p.User(model.NewContentImageURL(u), model.NewContentText("使用简洁清晰明确的一段中文纯文本描述图片"))
+				desc, err := api.Request(m)
+				if err != nil {
+					continue
+				}
+				msgs[i].Data["__agent_desc__"] = desc
+				hasset = true
+			}
+			if hasset {
+				raw, err := json.Marshal(&msgs)
+				if err == nil {
+					ev.Message = raw
+				}
+			}
+		}
+	})
+}
+
 // GetAction get OneBot CallAction from LLM and add it to context.
 //
 // Note:

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7
-	github.com/fumiama/deepinfra v0.0.0-20250920170049-e3d1b92cc3a1
+	github.com/fumiama/deepinfra v0.0.0-20250924134153-348633b5c6fb
 	github.com/pkg/errors v0.9.1
 	github.com/wdvxdr1123/ZeroBot v1.8.2-0.20250921063512-13752a73d444
 	gopkg.in/yaml.v3 v3.0.1
@@ -12,9 +12,11 @@ require (
 
 require (
 	github.com/FloatTech/ttl v0.0.0-20240716161252-965925764562 // indirect
+	github.com/fumiama/imgsz v0.0.4 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/tidwall/gjson v1.17.3 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
+	golang.org/x/image v0.16.0 // indirect
 	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
 )

go.sum

@@ -5,8 +5,10 @@ github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7/go.mod h1:vD7Ra3Q9o
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fumiama/deepinfra v0.0.0-20250920170049-e3d1b92cc3a1 h1:6PglFpNVm3DalGyRldacW2/v4jGWwn3v3q1tr2PhbVQ=
-github.com/fumiama/deepinfra v0.0.0-20250920170049-e3d1b92cc3a1/go.mod h1:wW05PQSn8mo1mZIoa6LBUE+3xIBjkoONvnfPTV5ZOhY=
+github.com/fumiama/deepinfra v0.0.0-20250924134153-348633b5c6fb h1:d0gl31nyOER4O3K0wpepont8flrDtzZsKACeWGxARX0=
+github.com/fumiama/deepinfra v0.0.0-20250924134153-348633b5c6fb/go.mod h1:HgRu6n9XgeyNki7wHHlTa75ORiv3UlYZafqOAoazhOI=
+github.com/fumiama/imgsz v0.0.4 h1:Lsasu2hdSSFS+vnD+nvR1UkiRMK7hcpyYCC0FzgSMFI=
+github.com/fumiama/imgsz v0.0.4/go.mod h1:bISOQVTlw9sRytPwe8ir7tAaEmyz9hSNj9n8mXMBG0E=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -24,6 +26,8 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/wdvxdr1123/ZeroBot v1.8.2-0.20250921063512-13752a73d444 h1:7aYFXzvVr2zuxBvqrGaJb24Z4W12aXBdW8DuE1mteE4=
 github.com/wdvxdr1123/ZeroBot v1.8.2-0.20250921063512-13752a73d444/go.mod h1:C86nQ0gIdAri4K2vg8IIQIslt08zzrKMcqYt8zhkx1M=
+golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw=
+golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=