feat(excel):尝试增加Excel通用解析过程。

This commit is contained in:
徐涛 2022-08-17 14:00:40 +08:00
parent 0fbe021252
commit b1b4da996e
5 changed files with 214 additions and 2 deletions

161
excel/abstract.go Normal file
View File

@ -0,0 +1,161 @@
package excel
import (
"electricity_bill_calc/utils"
"fmt"
"io"
"reflect"
"strconv"
"strings"
"github.com/shopspring/decimal"
"github.com/xuri/excelize/v2"
)
type ColumnRecognizer struct {
Pattern []string
Tag string
MatchIndex int
}
type ExcelAnalyzer[T any] struct {
File *excelize.File
ActivedSheet string
Regconizers []ColumnRecognizer
}
type ExcelAnalysisError struct {
Row int
Col int
Err error
}
func NewColumnRecognizer(tag string, patterns ...string) ColumnRecognizer {
return ColumnRecognizer{
Pattern: patterns,
Tag: tag,
MatchIndex: -1,
}
}
func (r *ColumnRecognizer) Recognize(cellValue string) bool {
for _, p := range r.Pattern {
if strings.Contains(cellValue, p) {
return true
}
}
return false
}
func NewExcelAnalyzer[T any](file io.Reader, recognizers []ColumnRecognizer) (*ExcelAnalyzer[T], error) {
excelFile, err := excelize.OpenReader(file)
if err != nil {
return nil, err
}
sheets := excelFile.GetSheetList()
return &ExcelAnalyzer[T]{
File: excelFile,
ActivedSheet: sheets[0],
Regconizers: recognizers,
}, nil
}
func (a *ExcelAnalyzer[T]) AddRecognizer(recognizer ColumnRecognizer) {
a.Regconizers = append(a.Regconizers, recognizer)
}
func (a *ExcelAnalyzer[T]) analysisTitleRow(cells []string) {
for col, content := range cells {
for _, recognizer := range a.Regconizers {
if recognizer.Recognize(content) {
recognizer.MatchIndex = col
break
}
}
}
}
func (a *ExcelAnalyzer[T]) Analysis(bean T) ([]T, []ExcelAnalysisError) {
errs := make([]ExcelAnalysisError, 0)
rows, err := a.File.GetRows(a.ActivedSheet)
if err != nil {
errs = append(errs, ExcelAnalysisError{Row: 0, Col: 0, Err: err})
return make([]T, 0), errs
}
elementType := reflect.TypeOf(bean)
collections := reflect.MakeSlice(elementType, 0, 0)
for rowIndex, cols := range rows {
// 标题行,需要完成识别动作
if rowIndex == 0 {
a.analysisTitleRow(cols)
continue
}
// 非标题行,创建一个需要收集的目标实例,然后逐识别器开始赋值
instance := reflect.New(elementType)
for i := 0; i < elementType.NumField(); i++ {
field := elementType.Field(i)
// 循环目标实例中的字段,与识别器匹配以后,取出指定列的值
if alias, ok := field.Tag.Lookup("excel"); ok {
for _, recognizer := range a.Regconizers {
if alias == recognizer.Tag && recognizer.MatchIndex != -1 {
actualField := instance.FieldByName(field.Name)
matchValue := cols[recognizer.MatchIndex]
switch field.Type.String() {
case "string":
actualField.Set(reflect.ValueOf(matchValue))
case "*string":
if len(matchValue) > 0 {
actualField.Elem().Set(reflect.ValueOf(&matchValue))
}
case "decimal.Decimal":
decimalValue, err := decimal.NewFromString(matchValue)
if err != nil {
errs = append(errs, ExcelAnalysisError{Row: rowIndex + 1, Col: recognizer.MatchIndex + 1, Err: fmt.Errorf("单元格内容应为纯数字内容。%w", err)})
actualField.Set(reflect.ValueOf(decimal.Zero))
} else {
actualField.Set(reflect.ValueOf(decimalValue))
}
case "decimal.NullDecimal":
nullValue := decimal.NewNullDecimal(decimal.Zero)
nullValue.Valid = false
if len(matchValue) == 0 {
actualField.Set(reflect.ValueOf(nullValue))
} else {
decimalValue, err := decimal.NewFromString(matchValue)
if err != nil {
errs = append(errs, ExcelAnalysisError{Row: rowIndex + 1, Col: recognizer.MatchIndex + 1, Err: fmt.Errorf("单元格内容应为纯数字内容。%w", err)})
actualField.Set(reflect.ValueOf((nullValue)))
} else {
actualField.Set(reflect.ValueOf(decimalValue))
}
}
case "int64", "int":
if len(matchValue) == 0 {
actualField.SetInt(0)
} else {
v, err := strconv.Atoi(matchValue)
if err != nil {
errs = append(errs, ExcelAnalysisError{Row: rowIndex + 1, Col: recognizer.MatchIndex + 1, Err: fmt.Errorf("单元格内容应为不带小鼠的整数。%w", err)})
actualField.SetInt(0)
} else {
actualField.SetInt(int64(v))
}
}
case "bool":
if utils.ContainsInsensitive(matchValue, []string{"是", "yes", "y", "true", "t"}) {
actualField.SetBool(true)
} else {
actualField.SetBool(false)
}
}
}
}
}
}
collections = reflect.Append(collections, instance.Elem())
}
return collections.Interface().([]T), errs
}

22
excel/meter_archive.go Normal file
View File

@ -0,0 +1,22 @@
package excel
import (
"electricity_bill_calc/model"
"io"
)
var meter04kVExcelRecognizers = []ColumnRecognizer{
{Pattern: []string{"表号"}, Tag: "code", MatchIndex: -1},
{Pattern: []string{"户名"}, Tag: "name", MatchIndex: -1},
{Pattern: []string{"户址"}, Tag: "address", MatchIndex: -1},
{Pattern: []string{"联系人"}, Tag: "contact", MatchIndex: -1},
{Pattern: []string{"电话"}, Tag: "phone", MatchIndex: -1},
{Pattern: []string{"倍率"}, Tag: "ratio", MatchIndex: -1},
{Pattern: []string{"序号"}, Tag: "seq", MatchIndex: -1},
{Pattern: []string{"公共", "公共设备"}, Tag: "public", MatchIndex: -1},
{Pattern: []string{"摊薄"}, Tag: "dilute", MatchIndex: -1},
}
func NewMeterArchiveExcelAnalyzer(file io.Reader) (*ExcelAnalyzer[model.Meter04KV], error) {
return NewExcelAnalyzer[model.Meter04KV](file, meter04kVExcelRecognizers)
}

10
go.mod
View File

@ -7,10 +7,13 @@ require (
github.com/go-redis/redis/v8 v8.11.5
github.com/google/uuid v1.3.0
github.com/jackc/pgx/v5 v5.0.0-beta.1
github.com/jinzhu/copier v0.3.5
github.com/liamylian/jsontime/v2 v2.0.0
github.com/mozillazg/go-pinyin v0.19.0
github.com/shopspring/decimal v1.3.1
github.com/spf13/viper v1.12.0
github.com/vmihailenco/msgpack/v5 v5.3.5
github.com/xuri/excelize/v2 v2.6.0
xorm.io/builder v0.3.12
xorm.io/xorm v1.3.1
)
@ -28,7 +31,6 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jinzhu/copier v0.3.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/magiconair/properties v1.8.6 // indirect
@ -36,9 +38,11 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mozillazg/go-pinyin v0.19.0 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.2 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.1 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
@ -47,6 +51,8 @@ require (
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/ugorji/go/codec v1.2.7 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8 // indirect
github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
golang.org/x/net v0.0.0-20220809012201-f428fae20770 // indirect
golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 // indirect

18
go.sum
View File

@ -387,6 +387,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/mozillazg/go-pinyin v0.19.0 h1:p+J8/kjJ558KPvVGYLvqBhxf8jbZA2exSLCs2uUVN8c=
github.com/mozillazg/go-pinyin v0.19.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@ -453,6 +455,10 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o=
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@ -520,6 +526,12 @@ github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8 h1:3X7aE0iLKJ5j+tz58BpvIZkXNV7Yq4jC93Z/rbN2Fxk=
github.com/xuri/efp v0.0.0-20220407160117-ad0f7a785be8/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.6.0 h1:m/aXAzSAqxgt74Nfd+sNzpzVKhTGl7+S9nbG4A57mF4=
github.com/xuri/excelize/v2 v2.6.0/go.mod h1:Q1YetlHesXEKwGFfeJn7PfEZz2IvHb6wdOeYjBxVcVs=
github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 h1:OAmKAfT06//esDdpi/DZ8Qsdt4+M5+ltca05dA5bG2M=
github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -566,6 +578,7 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220408190544-5352b0902921/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -580,6 +593,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -642,6 +657,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220809012201-f428fae20770 h1:dIi4qVdvjZEjiMDv7vhokAZNGnz3kepwuXqFKYDdDMs=
golang.org/x/net v0.0.0-20220809012201-f428fae20770/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -718,11 +734,13 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs=
golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

View File

@ -15,6 +15,11 @@ func Contains[T string | int | uint](element T, slice []T) bool {
return false
}
func ContainsInsensitive(element string, slice []string) bool {
lowercasedElement := strings.TrimSpace(strings.ToLower(element))
return Contains(lowercasedElement, slice)
}
func PinyinAbbr(source string) string {
abbr := pinyin.Pinyin(source, pinyin.NewArgs())
var abbrCollect = make([]string, 0)