From b1b4da996e67bef202f330d69c604b5ea0efcfac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E6=B6=9B?= Date: Wed, 17 Aug 2022 14:00:40 +0800 Subject: [PATCH] =?UTF-8?q?feat(excel):=E5=B0=9D=E8=AF=95=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0Excel=E9=80=9A=E7=94=A8=E8=A7=A3=E6=9E=90=E8=BF=87?= =?UTF-8?q?=E7=A8=8B=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- excel/abstract.go | 161 +++++++++++++++++++++++++++++++++++++++++ excel/meter_archive.go | 22 ++++++ go.mod | 10 ++- go.sum | 18 +++++ utils/utils.go | 5 ++ 5 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 excel/abstract.go create mode 100644 excel/meter_archive.go diff --git a/excel/abstract.go b/excel/abstract.go new file mode 100644 index 0000000..b563e9a --- /dev/null +++ b/excel/abstract.go @@ -0,0 +1,161 @@ +package excel + +import ( + "electricity_bill_calc/utils" + "fmt" + "io" + "reflect" + "strconv" + "strings" + + "github.com/shopspring/decimal" + "github.com/xuri/excelize/v2" +) + +type ColumnRecognizer struct { + Pattern []string + Tag string + MatchIndex int +} + +type ExcelAnalyzer[T any] struct { + File *excelize.File + ActivedSheet string + Regconizers []ColumnRecognizer +} + +type ExcelAnalysisError struct { + Row int + Col int + Err error +} + +func NewColumnRecognizer(tag string, patterns ...string) ColumnRecognizer { + return ColumnRecognizer{ + Pattern: patterns, + Tag: tag, + MatchIndex: -1, + } +} + +func (r *ColumnRecognizer) Recognize(cellValue string) bool { + for _, p := range r.Pattern { + if strings.Contains(cellValue, p) { + return true + } + } + return false +} + +func NewExcelAnalyzer[T any](file io.Reader, recognizers []ColumnRecognizer) (*ExcelAnalyzer[T], error) { + excelFile, err := excelize.OpenReader(file) + if err != nil { + return nil, err + } + sheets := excelFile.GetSheetList() + return &ExcelAnalyzer[T]{ + File: excelFile, + ActivedSheet: sheets[0], + Regconizers: recognizers, + }, nil +} + +func (a *ExcelAnalyzer[T]) AddRecognizer(recognizer ColumnRecognizer) { + a.Regconizers = append(a.Regconizers, recognizer) +} + +func (a *ExcelAnalyzer[T]) analysisTitleRow(cells []string) { + for col, content := range cells { + for _, recognizer := range a.Regconizers { + if recognizer.Recognize(content) { + recognizer.MatchIndex = col + break + } + } + } +} + +func (a *ExcelAnalyzer[T]) Analysis(bean T) ([]T, []ExcelAnalysisError) { + errs := make([]ExcelAnalysisError, 0) + + rows, err := a.File.GetRows(a.ActivedSheet) + if err != nil { + errs = append(errs, ExcelAnalysisError{Row: 0, Col: 0, Err: err}) + return make([]T, 0), errs + } + + elementType := reflect.TypeOf(bean) + collections := reflect.MakeSlice(elementType, 0, 0) + for rowIndex, cols := range rows { + // 标题行，需要完成识别动作 + if rowIndex == 0 { + a.analysisTitleRow(cols) + continue + } + // 非标题行，创建一个需要收集的目标实例，然后逐识别器开始赋值 + instance := reflect.New(elementType) + for i := 0; i < elementType.NumField(); i++ { + field := elementType.Field(i) + // 循环目标实例中的字段，与识别器匹配以后，取出指定列的值 + if alias, ok := field.Tag.Lookup("excel"); ok { + for _, recognizer := range a.Regconizers { + if alias == recognizer.Tag && recognizer.MatchIndex != -1 { + actualField := instance.FieldByName(field.Name) + matchValue := cols[recognizer.MatchIndex] + switch field.Type.String() { + case "string": + actualField.Set(reflect.ValueOf(matchValue)) + case "*string": + if len(matchValue) > 0 { + actualField.Elem().Set(reflect.ValueOf(&matchValue)) + } + case "decimal.Decimal": + decimalValue, err := decimal.NewFromString(matchValue) + if err != nil { + errs = append(errs, ExcelAnalysisError{Row: rowIndex + 1, Col: recognizer.MatchIndex + 1, Err: fmt.Errorf("单元格内容应为纯数字内容。%w", err)}) + actualField.Set(reflect.ValueOf(decimal.Zero)) + } else { + actualField.Set(reflect.ValueOf(decimalValue)) + } + case "decimal.NullDecimal": + nullValue := decimal.NewNullDecimal(decimal.Zero) + nullValue.Valid = false + if len(matchValue) == 0 { + actualField.Set(reflect.ValueOf(nullValue)) + } else { + decimalValue, err := decimal.NewFromString(matchValue) + if err != nil { + errs = append(errs, ExcelAnalysisError{Row: rowIndex + 1, Col: recognizer.MatchIndex + 1, Err: fmt.Errorf("单元格内容应为纯数字内容。%w", err)}) + actualField.Set(reflect.ValueOf((nullValue))) + } else { + actualField.Set(reflect.ValueOf(decimalValue)) + } + } + case "int64", "int": + if len(matchValue) == 0 { + actualField.SetInt(0) + } else { + v, err := strconv.Atoi(matchValue) + if err != nil { + errs = append(errs, ExcelAnalysisError{Row: rowIndex + 1, Col: recognizer.MatchIndex + 1, Err: fmt.Errorf("单元格内容应为不带小鼠的整数。%w", err)}) + actualField.SetInt(0) + } else { + actualField.SetInt(int64(v)) + } + } + case "bool": + if utils.ContainsInsensitive(matchValue, []string{"是", "yes", "y", "true", "t"}) { + actualField.SetBool(true) + } else { + actualField.SetBool(false) + } + } + } + } + } + } + collections = reflect.Append(collections, instance.Elem()) + } + + return collections.Interface().([]T), errs +} diff --git a/excel/meter_archive.go b/excel/meter_archive.go new file mode 100644 index 0000000..7ad3200 --- /dev/null +++ b/excel/meter_archive.go @@ -0,0 +1,22 @@ +package excel + +import ( + "electricity_bill_calc/model" + "io" +) + +var meter04kVExcelRecognizers = []ColumnRecognizer{ + {Pattern: []string{"表号"}, Tag: "code", MatchIndex: -1}, + {Pattern: []string{"户名"}, Tag: "name", MatchIndex: -1}, + {Pattern: []string{"户址"}, Tag: "address", MatchIndex: -1}, + {Pattern: []string{"联系人"}, Tag: "contact", MatchIndex: -1}, + {Pattern: []string{"电话"}, Tag: "phone", MatchIndex: -1}, + {Pattern: []string{"倍率"}, Tag: "ratio", MatchIndex: -1}, + {Pattern: []string{"序号"}, Tag: "seq", MatchIndex: -1}, + {Pattern: []string{"公共", "公共设备"}, Tag: "public", MatchIndex: -1}, + {Pattern: []string{"摊薄"}, Tag: "dilute", MatchIndex: -1}, +} + +func NewMeterArchiveExcelAnalyzer(file io.Reader) (*ExcelAnalyzer[model.Meter04KV], error) { + return NewExcelAnalyzer[model.Meter04KV](file, meter04kVExcelRecognizers) +} diff --git a/go.mod b/go.mod index 32c1b4e..1b96598 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,13 @@ require ( github.com/go-redis/redis/v8 v8.11.5 github.com/google/uuid v1.3.0 github.com/jackc/pgx/v5 v5.0.0-beta.1 + github.com/jinzhu/copier v0.3.5 github.com/liamylian/jsontime/v2 v2.0.0 + github.com/mozillazg/go-pinyin v0.19.0 github.com/shopspring/decimal v1.3.1 github.com/spf13/viper v1.12.0 github.com/vmihailenco/msgpack/v5 v5.3.5 + github.com/xuri/excelize/v2 v2.6.0 xorm.io/builder v0.3.12 xorm.io/xorm v1.3.1 ) @@ -28,7 +31,6 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jinzhu/copier v0.3.5 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/magiconair/properties v1.8.6 // indirect @@ -36,9 +38,11 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mozillazg/go-pinyin v0.19.0 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.1 // indirect github.com/spf13/afero v1.8.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -47,6 +51,8 @@ require ( github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ugorji/go/codec v1.2.7 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8 // indirect + github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/net v0.0.0-20220809012201-f428fae20770 // indirect golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 // indirect diff --git a/go.sum b/go.sum index 508969c..e9ebf2b 100644 --- a/go.sum +++ b/go.sum @@ -387,6 +387,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mozillazg/go-pinyin v0.19.0 h1:p+J8/kjJ558KPvVGYLvqBhxf8jbZA2exSLCs2uUVN8c= github.com/mozillazg/go-pinyin v0.19.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -453,6 +455,10 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -520,6 +526,12 @@ github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8 h1:3X7aE0iLKJ5j+tz58BpvIZkXNV7Yq4jC93Z/rbN2Fxk= +github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.6.0 h1:m/aXAzSAqxgt74Nfd+sNzpzVKhTGl7+S9nbG4A57mF4= +github.com/xuri/excelize/v2 v2.6.0/go.mod h1:Q1YetlHesXEKwGFfeJn7PfEZz2IvHb6wdOeYjBxVcVs= +github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 h1:OAmKAfT06//esDdpi/DZ8Qsdt4+M5+ltca05dA5bG2M= +github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -566,6 +578,7 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220408190544-5352b0902921/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -580,6 +593,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -642,6 +657,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220809012201-f428fae20770 h1:dIi4qVdvjZEjiMDv7vhokAZNGnz3kepwuXqFKYDdDMs= golang.org/x/net v0.0.0-20220809012201-f428fae20770/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -718,11 +734,13 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs= golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/utils/utils.go b/utils/utils.go index 66b76ba..2776a0a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -15,6 +15,11 @@ func Contains[T string | int | uint](element T, slice []T) bool { return false } +func ContainsInsensitive(element string, slice []string) bool { + lowercasedElement := strings.TrimSpace(strings.ToLower(element)) + return Contains(lowercasedElement, slice) +} + func PinyinAbbr(source string) string { abbr := pinyin.Pinyin(source, pinyin.NewArgs()) var abbrCollect = make([]string, 0)