From ad46584760d78cc40bd49c243222d3346319d8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?= <41315874+fumiama@users.noreply.github.com> Date: Thu, 4 May 2023 23:05:40 +0800 Subject: [PATCH] finish chkdup --- backend/file.go | 164 +++++++++++++- backend/global/file.go | 19 +- backend/global/list.go | 6 +- backend/global/question.go | 2 +- backend/question.go | 72 ++++-- frontend/vben/src/api/page/index.ts | 13 +- frontend/vben/src/api/page/model/fileModel.ts | 9 + frontend/vben/src/views/page/chkdup/data.ts | 14 ++ frontend/vben/src/views/page/chkdup/index.vue | 208 +++++++++++++++++- 9 files changed, 466 insertions(+), 41 deletions(-) diff --git a/backend/file.go b/backend/file.go index 98e54d4..4cc0229 100644 --- a/backend/file.go +++ b/backend/file.go @@ -1,7 +1,10 @@ package backend import ( + "container/heap" + "encoding/json" "errors" + "math" "net/http" "strconv" "strings" @@ -31,8 +34,8 @@ type filelist struct { Per uint `json:"percent"` } -func getFileList(count int, istemp *bool) ([]filelist, error) { - lst, err := global.FileDB.ListUploadedFile(istemp) +func getFileList(count, uid int, istemp *bool) ([]filelist, error) { + lst, err := global.FileDB.ListUploadedFile(istemp, uid) if err != nil && err != sql.ErrNullResult { return nil, err } @@ -124,23 +127,127 @@ type filestatus struct { } func getFileStatus(lstid int, user *global.User) (*filestatus, error) { - file, sz, istemp, err := global.FileDB.GetFile(lstid, *user.ID) + file, lst, err := global.FileDB.GetFile(lstid, *user.ID) if err != nil { return nil, err } - qs, ds, filerate, err := parseFileQuestions(file.Questions, istemp) + qs, ds, filerate, err := parseFileQuestions(file.Questions, lst.IsTemp, true) if err != nil { return nil, err } return &filestatus{ Name: file.Class + ".docx", - Size: float64(sz) / 1024 / 1024, // MB + Size: float64(lst.Size) / 1024 / 1024, // MB Rate: filerate * 100, Questions: qs, Duplications: ds, }, nil } +type filestatusdup struct { + Name string `json:"name"` + Size float64 `json:"size"` + Rate int `json:"rate"` + Questions []question `json:"questions"` + Duplications []duplication `json:"duplications"` // Duplications length == 10 + Files []duplication `json:"files"` // Files is {dup: desc}[] in yearstart..yearend +} + +func checkFileDup(lstid int, user *global.User, yearstart, yearend global.StudyYear) (*filestatusdup, error) { + file, lst, err := global.FileDB.GetFile(lstid, *user.ID) + if err != nil { + return nil, err + } + files, err := global.FileDB.GetFilesByYearRange(yearstart, yearend) + if err != nil { + return nil, err + } + qs := make([]question, 0, 16) + filesdups := make([]duplication, 0, 16) + ques := make([]global.QuestionJSON, 0, 64) + myques := make([]global.QuestionJSON, 0, 64) + err = json.Unmarshal(file.Questions, &myques) + if err != nil { + return nil, err + } + dh := make(duplications, 0, 16) + heap.Init(&dh) + filerate := 0 + isfirstmy := true + totl := 0 + for _, f := range files { + if f.ListID == lstid { + continue + } + ques = ques[:0] + err := json.Unmarshal(f.Questions, &ques) + if err != nil { + return nil, err + } + sum := 0.0 + cnt := 0 + for _, q := range myques { + if isfirstmy { + qs = append(qs, question{ + Count: len(q.Sub), + Point: q.Points, + Name: q.Name, + }) + } + for i, subq := range q.Sub { + p, err := getQuestionDupFromPaper(subq, ques, lst.IsTemp) + if err != nil { + return nil, err + } + heap.Push(&dh, duplication{ + Percent: int(math.Round(p * 100)), + Name: q.Name + "." + strconv.Itoa(i+1), + }) + sum += p + cnt++ + } + } + isfirstmy = false + pc := int(math.Round(sum * 100 / float64(cnt))) + flst, err := f.GetList(&global.FileDB) + name := "" + if err != nil { + name = err.Error() + } else { + name = flst.Desc + } + filesdups = append(filesdups, duplication{ + Percent: pc, + Name: name, + }) + filerate += pc + totl++ + } + if totl == 0 { + return nil, sql.ErrNullResult + } + i := dh.Len() + ds := make([]duplication, 10) + if i > 10 { + i = 10 + } else { + for j := i; j < 10; j++ { + ds[j] = duplication{Name: "N/A"} + } + } + for i--; i >= 0; i-- { + ds[i] = heap.Pop(&dh).(duplication) + } + return &filestatusdup{ + Name: file.Class + ".docx", + Size: float64(lst.Size) / 1024 / 1024, // MB + Rate: filerate / totl, + Questions: qs, + Duplications: ds, + Files: filesdups, + }, nil +} + func init() { apimap["/api/getFileList"] = &apihandler{"GET", func(w http.ResponseWriter, r *http.Request) { token := r.Header.Get("Authorization") @@ -165,7 +272,7 @@ func init() { return } } - lst, err := getFileList(count, istemp) + lst, err := getFileList(count, *user.ID, istemp) if err != nil { writeresult(w, codeError, nil, err.Error(), typeError) return @@ -281,6 +388,51 @@ func init() { } writeresult(w, codeSuccess, fstat, messageOk, typeSuccess) }} + + apimap["/api/checkFileDup"] = &apihandler{"GET", func(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("Authorization") + user := usertokens.Get(token) + if user == nil { + writeresult(w, codeError, nil, errInvalidToken.Error(), typeError) + return + } + idstr := r.URL.Query().Get("id") + if idstr == "" { + writeresult(w, codeError, nil, "empty id", typeError) + return + } + id, err := strconv.Atoi(idstr) + if err != nil { + writeresult(w, codeError, nil, err.Error(), typeError) + return + } + ysstr := r.URL.Query().Get("ys") + if ysstr == "" { + writeresult(w, codeError, nil, "empty ys", typeError) + return + } + ys, err := strconv.Atoi(ysstr) + if err != nil { + writeresult(w, codeError, nil, err.Error(), typeError) + return + } + yestr := r.URL.Query().Get("ye") + if yestr == "" { + writeresult(w, codeError, nil, "empty ye", typeError) + return + } + ye, err := strconv.Atoi(yestr) + if err != nil { + writeresult(w, codeError, nil, err.Error(), typeError) + return + } + fstat, err := checkFileDup(id, user, global.StudyYear(ys), global.StudyYear(ye)) + if err != nil { + writeresult(w, codeError, nil, err.Error(), typeError) + return + } + writeresult(w, codeSuccess, fstat, messageOk, typeSuccess) + }} } // FileHandler serves contents in global.FileFolder diff --git a/backend/global/file.go b/backend/global/file.go index 65df284..abb433f 100644 --- a/backend/global/file.go +++ b/backend/global/file.go @@ -98,6 +98,12 @@ func (sy StudyYear) String() string { return strconv.Itoa(int(sy)) + "-" + strconv.Itoa(int(next)) + "学年" } +func (file *File) GetList(f *FileDatabase) (List, error) { + f.mu.RLock() + defer f.mu.RUnlock() + return sql.Find[List](&f.db, FileTableList, "WHERE ID="+strconv.Itoa(file.ListID)) +} + // DelFile by listid func (f *FileDatabase) DelFile(lstid, uid int, istemp bool) error { user, err := UserDB.GetUserByID(uid) @@ -162,10 +168,10 @@ func (f *FileDatabase) DelFile(lstid, uid int, istemp bool) error { } // GetFile get analyzed file's structure from List(ID) -func (f *FileDatabase) GetFile(lstid, uid int) (file *File, sz int64, istemp bool, err error) { +func (f *FileDatabase) GetFile(lstid, uid int) (file *File, lst List, err error) { f.mu.RLock() defer f.mu.RUnlock() - lst, err := sql.Find[List](&f.db, FileTableList, "WHERE ID="+strconv.Itoa(lstid)) + lst, err = sql.Find[List](&f.db, FileTableList, "WHERE ID="+strconv.Itoa(lstid)) if err != nil { return } @@ -187,5 +193,12 @@ func (f *FileDatabase) GetFile(lstid, uid int) (file *File, sz int64, istemp boo if err != nil { return } - return &filestruct, lst.Size, lst.IsTemp, nil + return &filestruct, lst, nil +} + +// GetFilesByYearRange ... +func (f *FileDatabase) GetFilesByYearRange(yearstart, yearend StudyYear) ([]*File, error) { + f.mu.RLock() + defer f.mu.RUnlock() + return sql.FindAll[File](&f.db, FileTableFile, "WHERE Year>="+strconv.Itoa(int(yearstart))+" AND Year<="+strconv.Itoa(int(yearend))) } diff --git a/backend/global/list.go b/backend/global/list.go index 25f50c5..796952a 100644 --- a/backend/global/list.go +++ b/backend/global/list.go @@ -83,13 +83,13 @@ func (f *FileDatabase) SaveFileToTemp(uploader int, file io.Reader, name string) } // ListUploadedFile will select all file that HasntAnalyzed && IsTemp or !HasntAnalyzed && !IsTemp -func (f *FileDatabase) ListUploadedFile(istemp *bool) (lst []*List, err error) { +func (f *FileDatabase) ListUploadedFile(istemp *bool, uid int) (lst []*List, err error) { q := "" switch { case istemp == nil: - q = "ORDER BY UpTime DESC" + q = "WHERE (NOT IsTemp) OR (Uploader=" + strconv.Itoa(uid) + ") ORDER BY UpTime DESC" case *istemp: - q = "WHERE IsTemp ORDER BY UpTime DESC" + q = "WHERE IsTemp AND Uploader=" + strconv.Itoa(uid) + " ORDER BY UpTime DESC" default: q = "WHERE (HasntAnalyzed AND IsTemp) OR (NOT HasntAnalyzed AND NOT IsTemp) ORDER BY UpTime DESC" } diff --git a/backend/global/question.go b/backend/global/question.go index 0bb5b8a..e10139b 100644 --- a/backend/global/question.go +++ b/backend/global/question.go @@ -103,7 +103,7 @@ func (f *FileDatabase) GetQuestion(id int64, istemp bool) (Question, error) { } // GetQuestionHex by hexid -func (f *FileDatabase) GetQuestionHex(hexid string, istemp bool) (q Question, err error) { +func (f *FileDatabase) GetQuestionByHex(hexid string, istemp bool) (q Question, err error) { idb, err := hex.DecodeString(hexid) if err != nil { return diff --git a/backend/question.go b/backend/question.go index 54fc99b..1a0c9c5 100644 --- a/backend/question.go +++ b/backend/question.go @@ -50,15 +50,19 @@ func (d *duplications) Pop() any { return x } -func parseFileQuestions(qb []byte, istemp bool) ([]question, []duplication, float64, error) { +func parseFileQuestions(qb []byte, istemp, getdup bool) ([]question, []duplication, float64, error) { ques := make([]global.QuestionJSON, 0, 16) qs := make([]question, 0, 16) err := json.Unmarshal(qb, &ques) if err != nil { return nil, nil, 0, err } - dh := make(duplications, 0, 16) - heap.Init(&dh) + dhp := (*duplications)(nil) + if getdup { + dh := make(duplications, 0, 16) + heap.Init(&dh) + dhp = &dh + } sum := 0.0 cnt := 0 for _, q := range ques { @@ -68,33 +72,63 @@ func parseFileQuestions(qb []byte, istemp bool) ([]question, []duplication, floa Name: q.Name, }) for i, subq := range q.Sub { - qstruct, err := global.FileDB.GetQuestionHex(subq.Name, istemp) + qstruct, err := global.FileDB.GetQuestionByHex(subq.Name, istemp) if err != nil { continue } p := qstruct.MaxDuplicateRate() - heap.Push(&dh, duplication{ - Percent: int(math.Round(p * 100)), - Name: q.Name + "." + strconv.Itoa(i+1), - }) + if getdup { + heap.Push(dhp, duplication{ + Percent: int(math.Round(p * 100)), + Name: q.Name + "." + strconv.Itoa(i+1), + }) + } sum += p cnt++ } } - i := dh.Len() - ds := make([]duplication, 10) - if i > 10 { - i = 10 - } else { - for j := i; j < 10; j++ { - ds[j] = duplication{Name: "N/A"} + if getdup { + i := dhp.Len() + ds := make([]duplication, 10) + if i > 10 { + i = 10 + } else { + for j := i; j < 10; j++ { + ds[j] = duplication{Name: "N/A"} + } } - } - for i--; i >= 0; i-- { - ds[i] = heap.Pop(&dh).(duplication) + for i--; i >= 0; i-- { + ds[i] = heap.Pop(dhp).(duplication) + } + return qs, ds, sum / float64(cnt), nil } - return qs, ds, sum / float64(cnt), nil + return qs, nil, sum / float64(cnt), nil +} + +// getQuestionDupFromPaper returns rate, error +func getQuestionDupFromPaper(que global.QuestionJSON, ques []global.QuestionJSON, istemp bool) (float64, error) { + myqstruct, err := global.FileDB.GetQuestionByHex(que.Name, istemp) + if err != nil { + return -1, err + } + maxp := 0.0 + for _, q := range ques { + for _, subq := range q.Sub { + qstruct, err := global.FileDB.GetQuestionByHex(subq.Name, istemp) + if err != nil { + continue + } + p, err := myqstruct.GetDuplicateRate(&qstruct) + if err != nil { + continue + } + if maxp < p { + maxp = p + } + } + } + return maxp, nil } func init() { diff --git a/frontend/vben/src/api/page/index.ts b/frontend/vben/src/api/page/index.ts index 478a1cb..6855e3e 100644 --- a/frontend/vben/src/api/page/index.ts +++ b/frontend/vben/src/api/page/index.ts @@ -5,7 +5,7 @@ import { FileListGroupItem, GenerateConfig, } from './model/fileListModel' -import { DownloadFile, FileStatus } from './model/fileModel' +import { DownloadFile, FileDupStatus, FileStatus } from './model/fileModel' enum Api { GetFileList = '/getFileList', @@ -15,6 +15,7 @@ enum Api { AnalyzeFile = '/analyzeFile', DlFile = '/dlFile', GetFileStatus = '/getFileStatus', + CheckFileDup = '/checkFileDup', GetMajors = '/getMajors', GenFile = '/genFile', DlGen = '/dlGen', @@ -95,6 +96,16 @@ export const getFileStatus = (id: number) => { return defHttp.get({ url: Api.GetFileStatus, params: { id: id } }) } +/** + * @description: Check file duplication + */ +export const checkFileDup = (id: number, ys: number, ye: number) => { + return defHttp.get( + { url: Api.CheckFileDup, params: { id, ys, ye } }, + { errorMessageMode: 'none' }, + ) +} + /** * @description: Get majors */ diff --git a/frontend/vben/src/api/page/model/fileModel.ts b/frontend/vben/src/api/page/model/fileModel.ts index 15847ba..a19b405 100644 --- a/frontend/vben/src/api/page/model/fileModel.ts +++ b/frontend/vben/src/api/page/model/fileModel.ts @@ -20,3 +20,12 @@ export interface FileStatus { questions: Question[] duplications: Duplication[] } + +export interface FileDupStatus { + name: string + size: number + rate: number + questions: Question[] + duplications: Duplication[] + files: Duplication[] +} diff --git a/frontend/vben/src/views/page/chkdup/data.ts b/frontend/vben/src/views/page/chkdup/data.ts index 5a8661a..5439f44 100644 --- a/frontend/vben/src/views/page/chkdup/data.ts +++ b/frontend/vben/src/views/page/chkdup/data.ts @@ -1,4 +1,5 @@ import { FormSchema } from '/@/components/Form' +import { BasicColumn } from '/@/components/Table/src/types/table' export const schemas: FormSchema[] = [ { @@ -22,3 +23,16 @@ export const taskSchemas: FormSchema[] = [ }, }, ] +export const columns: BasicColumn[] = [ + { + title: '试卷', + dataIndex: 'name', + fixed: 'left', + width: 200, + }, + { + title: '重复率(%)', + dataIndex: 'percent', + width: 150, + }, +] diff --git a/frontend/vben/src/views/page/chkdup/index.vue b/frontend/vben/src/views/page/chkdup/index.vue index ed08cc1..2c413dc 100644 --- a/frontend/vben/src/views/page/chkdup/index.vue +++ b/frontend/vben/src/views/page/chkdup/index.vue @@ -24,8 +24,31 @@ - -

aaaaa

+ +
+ + +