Golang基准测试实战—测试读写大JSON和小JSON数据的读写性能

X_PENG
154 阅读3分钟

测试代码

main.go文件代码

import (
   "fmt"
   "gorm.io/driver/mysql"
   "gorm.io/gorm"
)

type TestPerformance2 struct {
   gorm.Model
   // 任务名称
   TaskName string `gorm:"column:task_name;type:varchar(255);not null;default:''"`
   // 任务配置，内容是JSON
   TaskConf string `gorm:"column:task_conf;type:text"`
   // 任务配置大小的等级，表示TaskConf会占用多少空间  0:未知 1:字节级别，不超过1kb 2:kb级别，不超过10kb
   TaskConfSizeLevel int `gorm:"column:task_conf_size_level;type:tinyint(4);not null;default:0"`
   // 创建人
   CreatedBy string `gorm:"column:created_by;type:varchar(30);default:''"`
   // 更新人
   UpdatedBy string `gorm:"column:updated_by;type:varchar(30);default:''"`
   // 删除人
   DeletedBy string `gorm:"column:deleted_by;type:varchar(30);not null;default:''"`
}

func (t *TestPerformance2) TableName() string {
   return "test_performance2"
}

var DB *gorm.DB
var (
   SmallTaskConf     = "{\n  "min_position": 9,\n  "has_more_items": false,\n  "items_html": "Bus",\n  "new_latent_count": 6,\n  "data": {\n    "length": 281,\n    "text": "Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."\n  },\n  "numericalArray": [\n    31\n  ],\n  "StringArray": [\n    "Oxygen"\n  ],\n  "multipleTypesArray": true,\n  "objArray": [\n    {\n      "class": "upper",\n      "age": 1\n    }\n  ]\n}"
   BigTaskConf       = "[\n  {\n    "_id": "63c2c841a6493f8217f2dc1e",\n    "index": 0,\n    "guid": "9b8d0eed-1480-432f-b3ab-5348639e4fe4",\n    "isActive": true,\n    "balance": "$1,565.42",\n    "picture": "http://placehold.it/32x32",\n    "age": 38,\n    "eyeColor": "brown",\n    "name": "Melton Kennedy",\n    "gender": "male",\n    "company": "COMVEYOR",\n    "email": "meltonkennedy@comveyor.com",\n    "phone": "+1 (800) 556-2723",\n    "address": "563 Bay Avenue, Russellville, Wyoming, 8142",\n    "about": "Sunt proident anim occaecat aute tempor aliqua aliqua excepteur. Nisi occaecat quis qui incididunt aliquip. Voluptate non id dolore dolor incididunt consectetur sit do sit occaecat irure minim ad. Anim consequat laborum ut culpa est adipisicing nulla cillum anim ut. Cupidatat pariatur cupidatat adipisicing esse ad. Eiusmod laboris deserunt qui labore dolore enim qui adipisicing eu non ea in.\r\n",\n    "registered": "2020-10-16T12:33:43 -08:00",\n    "latitude": -2.482562,\n    "longitude": -155.368238,\n    "tags": [\n      "dolore",\n      "consectetur",\n      "quis",\n      "in",\n      "nulla",\n      "duis",\n      "do"\n    ],\n    "friends": [\n      {\n        "id": 0,\n        "name": "Sheryl Snider"\n      },\n      {\n        "id": 1,\n        "name": "Juanita Petersen"\n      },\n      {\n        "id": 2,\n        "name": "Velez Bradley"\n      }\n    ],\n    "greeting": "Hello, Melton Kennedy! You have 1 unread messages.",\n    "favoriteFruit": "apple"\n  },\n  {\n    "_id": "63c2c841287d9a212798aa89",\n    "index": 1,\n    "guid": "9f9acbea-f95c-4ca1-9d1e-0370406314e7",\n    "isActive": true,\n    "balance": "$1,346.85",\n    "picture": "http://placehold.it/32x32",\n    "age": 33,\n    "eyeColor": "green",\n    "name": "Mitchell Marshall",\n    "gender": "male",\n    "company": "CODACT",\n    "email": "mitchellmarshall@codact.com",\n    "phone": "+1 (822) 533-2160",\n    "address": "621 Waldane Court, Rivers, Georgia, 7237",\n    "about": "Ullamco veniam id est deserunt ipsum quis ut. Ex enim ad reprehenderit voluptate dolore fugiat elit in cillum esse Lorem proident adipisicing. Occaecat voluptate aute id deserunt occaecat deserunt aute nulla sit aliquip Lorem.\r\n",\n    "registered": "2017-03-09T06:07:46 -08:00",\n    "latitude": 86.998893,\n    "longitude": -89.651276,\n    "tags": [\n      "culpa",\n      "consequat",\n      "veniam",\n      "laboris",\n      "Lorem",\n      "non",\n      "occaecat"\n    ],\n    "friends": [\n      {\n        "id": 0,\n        "name": "Leslie Roth"\n      },\n      {\n        "id": 1,\n        "name": "Edwards Graves"\n      },\n      {\n        "id": 2,\n        "name": "Farmer Bradshaw"\n      }\n    ],\n    "greeting": "Hello, Mitchell Marshall! You have 9 unread messages.",\n    "favoriteFruit": "apple"\n  },\n  {\n    "_id": "63c2c8418f7b5539810a23d8",\n    "index": 2,\n    "guid": "44e8ddab-5ea8-4524-a2b0-87350d80a815",\n    "isActive": false,\n    "balance": "$1,325.74",\n    "picture": "http://placehold.it/32x32",\n    "age": 29,\n    "eyeColor": "green",\n    "name": "Whitney Avila",\n    "gender": "male",\n    "company": "ENAUT",\n    "email": "whitneyavila@enaut.com",\n    "phone": "+1 (866) 536-3654",\n    "address": "579 Hoyts Lane, Felt, Alabama, 1794",\n    "about": "Nisi non enim Lorem officia elit incididunt fugiat in elit aliquip irure excepteur cillum. Duis minim cillum aliquip eiusmod dolore laboris pariatur commodo tempor anim consequat nostrud aute nulla. Commodo Lorem do ullamco eu.\r\n",\n    "registered": "2016-04-22T06:49:20 -08:00",\n    "latitude": -50.07122,\n    "longitude": 13.892166,\n    "tags": [\n      "adipisicing",\n      "consequat",\n      "et",\n      "voluptate",\n      "nisi",\n      "commodo",\n      "ut"\n    ],\n    "friends": [\n      {\n        "id": 0,\n        "name": "Duran Dunlap"\n      },\n      {\n        "id": 1,\n        "name": "Nelda Petty"\n      },\n      {\n        "id": 2,\n        "name": "Langley Santos"\n      }\n    ],\n    "greeting": "Hello, Whitney Avila! You have 9 unread messages.",\n    "favoriteFruit": "strawberry"\n  },\n  {\n    "_id": "63c2c8414d6684da3dde20c5",\n    "index": 3,\n    "guid": "e10e0b98-ffa5-4fef-a633-3b2a548537a1",\n    "isActive": false,\n    "balance": "$1,658.77",\n    "picture": "http://placehold.it/32x32",\n    "age": 38,\n    "eyeColor": "blue",\n    "name": "Salas Joyce",\n    "gender": "male",\n    "company": "VANTAGE",\n    "email": "salasjoyce@vantage.com",\n    "phone": "+1 (903) 496-2653",\n    "address": "786 Holly Street, Grazierville, Iowa, 5557",\n    "about": "Aute non do quis tempor non laborum ex reprehenderit ut culpa ad. Elit non occaecat tempor do amet officia ipsum ad amet. Elit reprehenderit eiusmod enim id eiusmod consequat irure cupidatat mollit aute nostrud. Reprehenderit incididunt cillum commodo eiusmod pariatur elit non qui officia.\r\n",\n    "registered": "2014-03-30T10:31:07 -08:00",\n    "latitude": 64.29487,\n    "longitude": -72.849846,\n    "tags": [\n      "cillum",\n      "consequat",\n      "laboris",\n      "occaecat",\n      "aliquip",\n      "nisi",\n      "irure"\n    ],\n    "friends": [\n      {\n        "id": 0,\n        "name": "Barr Dudley"\n      },\n      {\n        "id": 1,\n        "name": "Hart Mejia"\n      },\n      {\n        "id": 2,\n        "name": "Sandy Leblanc"\n      }\n    ],\n    "greeting": "Hello, Salas Joyce! You have 5 unread messages.",\n    "favoriteFruit": "apple"\n  },\n  {\n    "_id": "63c2c84119e79c46569e8fbb",\n    "index": 4,\n    "guid": "53f90410-fedf-42ff-a29a-989bd859fff1",\n    "isActive": true,\n    "balance": "$1,008.27",\n    "picture": "http://placehold.it/32x32",\n    "age": 24,\n    "eyeColor": "blue",\n    "name": "Reilly Schwartz",\n    "gender": "male",\n    "company": "IDETICA",\n    "email": "reillyschwartz@idetica.com",\n    "phone": "+1 (848) 509-2570",\n    "address": "662 Pitkin Avenue, Edgar, North Dakota, 5162",\n    "about": "Esse magna cillum dolore qui dolor elit culpa sit officia in adipisicing tempor quis fugiat. Consectetur nulla et esse officia adipisicing est esse tempor esse deserunt officia aute. Deserunt in culpa veniam laboris velit mollit sint adipisicing tempor sint non reprehenderit sint eiusmod. Excepteur aliqua amet ad voluptate.\r\n",\n    "registered": "2017-01-28T12:43:51 -08:00",\n    "latitude": 4.950632,\n    "longitude": -52.579471,\n    "tags": [\n      "ipsum",\n      "incididunt",\n      "cupidatat",\n      "in",\n      "sunt",\n      "ipsum",\n      "nulla"\n    ],\n    "friends": [\n      {\n        "id": 0,\n        "name": "Enid Duke"\n      },\n      {\n        "id": 1,\n        "name": "Gamble Hudson"\n      },\n      {\n        "id": 2,\n        "name": "Sanchez Schneider"\n      }\n    ],\n    "greeting": "Hello, Reilly Schwartz! You have 7 unread messages.",\n    "favoriteFruit": "apple"\n  }\n]"
   TaskNameTemplate  = "测试测试测试任务名称%d"
   CreatedByTemplate = "TEST-%d"
)

func init() {
   // TODO 你的mysql地址
   dsn := "root:xxxxxx@tcp(xxxxxxx:3306)/xxx?charset=utf8mb4&parseTime=True&loc=Local"
   var err error
   DB, err = gorm.Open(mysql.Open(dsn), &gorm.Config{})

   if err != nil {
      panic(err)
   }
}

func main() {
   DB.AutoMigrate(TestPerformance2{})

   rows, err := FindSmallJSONData(100)
   if err != nil {
      panic(err)
   }
   fmt.Println("FindSmallJSONData rows:", rows)

   rows, err = FindBigJSONData(100)
   if err != nil {
      panic(err)
   }
   fmt.Println("FindBigJSONData rows:", rows)
}

func BatchCreate(datas []*TestPerformance2) (int64, error) {
   res := DB.CreateInBatches(datas, 50)
   return res.RowsAffected, res.Error
}

// FindSmallJSONData 查询小JSON数据
func FindSmallJSONData(limit int) (int, error) {
   res := make([]*TestPerformance2, 0, 100)
   find := DB.Where("task_conf_size_level = ?", 1).Limit(limit).Find(&res)
   return len(res), find.Error
}

// FindBigJSONData 查询大JSON数据
func FindBigJSONData(limit int) (int, error) {
   res := make([]*TestPerformance2, 0, 100)
   find := DB.Where("task_conf_size_level = ?", 2).Limit(limit).Find(&res)
   return len(res), find.Error
}

main_test.go文件代码

import (
   "fmt"
   "testing"
)

var amount = 300
var amountInt64 int64 = 300

func TestSize(t *testing.T) {
   fmt.Println(len(SmallTaskConf))      // 432字节
   fmt.Println(len(BigTaskConf) / 1024) // 6kb
}

// go test -v -bench=BenchmarkBatchCreateSmallData -run=^$ -benchtime=10x ./
/*
测试结果：
BenchmarkBatchCreateSmallData
BenchmarkBatchCreateSmallData-12              10         709647276 ns/op
PASS
ok      go_orm_demo_1/t1        9.000s
*/
func BenchmarkBatchCreateSmallData(b *testing.B) {
   for i := 0; i < b.N; i++ {
      b.StopTimer()
      smallDatas := GetSmallDatas(i + 1)
      b.StartTimer()

      rows, err := BatchCreate(smallDatas)
      if err != nil {
         b.Fatalf("BenchmarkBatchCreateSmallData err:%v\n", err)
      }
      if rows != amountInt64 {
         b.Fatalf("BenchmarkBatchCreateSmallData BatchCreate rows:%d\n", rows)
      }
   }
}
func GetSmallDatas(count int) []*TestPerformance2 {
   start := (count - 1) * amount
   end := count * amount

   datas := make([]*TestPerformance2, 0, amount)
   for i := start; i < end; i++ {
      datas = append(datas, &TestPerformance2{
         TaskName:          fmt.Sprintf(TaskNameTemplate, i),
         TaskConf:          SmallTaskConf,
         TaskConfSizeLevel: 1,
         CreatedBy:         fmt.Sprintf(CreatedByTemplate, i),
      })
   }
   return datas
}

// go test -v -bench=BenchmarkBatchCreateBigData -run=^$ -benchtime=10x ./
/*
测试结果：
BenchmarkBatchCreateBigData-12                10         937207817 ns/op
PASS
ok      go_orm_demo_1/t1        12.698s
*/
func BenchmarkBatchCreateBigData(b *testing.B) {
   for i := 0; i < b.N; i++ {
      b.StopTimer()
      bigDatas := GetBigDatas(i + 1)
      b.StartTimer()

      rows, err := BatchCreate(bigDatas)
      if err != nil {
         b.Fatalf("BenchmarkBatchCreateBigData err:%v\n", err)
      }
      if rows != amountInt64 {
         b.Fatalf("BenchmarkBatchCreateBigData BatchCreate rows:%d\n", rows)
      }
   }
}

func GetBigDatas(count int) []*TestPerformance2 {
   start := (count - 1) * amount
   end := count * amount

   datas := make([]*TestPerformance2, 0, amount)
   for i := start; i < end; i++ {
      datas = append(datas, &TestPerformance2{
         TaskName:          fmt.Sprintf(TaskNameTemplate, i),
         TaskConf:          BigTaskConf,
         TaskConfSizeLevel: 2,
         CreatedBy:         fmt.Sprintf(CreatedByTemplate, i),
      })
   }
   return datas
}

// go test -v -bench=BenchmarkBatchGetSmallData -run=^$ -benchtime=10x ./
/*
测试结果：
BenchmarkBatchGetSmallData-12                 10         229965127 ns/op
PASS
ok      go_orm_demo_1/t1        3.478s
*/
func BenchmarkBatchGetSmallData(b *testing.B) {
   for i := 0; i < b.N; i++ {
      rows, err := FindSmallJSONData(100)
      if err != nil {
         b.Fatalf("BenchmarkBatchGetSmallData err:%v\n", err)
      }
      if rows != 100 {
         b.Fatalf("BenchmarkBatchGetSmallData rows:%d\n", rows)
      }
   }
}

// go test -v -bench=BenchmarkBatchGetBigData -run=^$ -benchtime=10x ./
/*
测试结果：
BenchmarkBatchGetBigData-12           10        2594930053 ns/op
PASS
ok      go_orm_demo_1/t1        28.913s
*/
func BenchmarkBatchGetBigData(b *testing.B) {
   for i := 0; i < b.N; i++ {
      rows, err := FindBigJSONData(100)
      if err != nil {
         b.Fatalf("BenchmarkBatchGetBigData err:%v\n", err)
      }
      if rows != 100 {
         b.Fatalf("BenchmarkBatchGetBigData rows:%d\n", rows)
      }
   }
}

测试写性能

操作步骤：

  1. 保证表数据为空，执行go test -v -bench=BenchmarkBatchCreateSmallData -run=^$ -benchtime=10x ./基准测试，看测试结果
  2. 保证表数据为空，执行go test -v -bench=BenchmarkBatchCreateBigData -run=^$ -benchtime=10x ./基准测试，看测试结果

测试结果：

BenchmarkBatchCreateSmallData
BenchmarkBatchCreateSmallData-12              10         709647276 ns/op
PASS
ok      go_orm_demo_1/t1        9.000s


BenchmarkBatchCreateBigData-12                10         937207817 ns/op
PASS
ok      go_orm_demo_1/t1        12.698s

可以看到，大JSON的写比小JSON的写多耗时230ms。

测试读性能

操作步骤：

  1. 保证表数据为空，执行go test -v -bench=BenchmarkBatchCreateSmallData -run=^$ -benchtime=10x ./和go test -v -bench=BenchmarkBatchCreateBigData -run=^$ -benchtime=10x ./造数据
  2. 执行go test -v -bench=BenchmarkBatchGetSmallData -run=^$ -benchtime=10x ./基准测试，看测试结果
  3. 执行go test -v -bench=BenchmarkBatchGetBigData -run=^$ -benchtime=10x ./基准测试，看测试结果

测试结果：

BenchmarkBatchGetSmallData-12                 10         229965127 ns/op
PASS
ok      go_orm_demo_1/t1        3.478s



BenchmarkBatchGetBigData-12           10        2594930053 ns/op
PASS
ok      go_orm_demo_1/t1        28.913s

可以看到，大JSON的读耗时2.5s，小JSON的读耗时229ms。

再执行SQL，把大JSON改成小JSON：

UPDATE test_performance2
set task_conf= $小JSON数据
where task_conf_size_level=2 // task_conf_size_level=2的数据是大JSON数据

然后，再执行下基准测试go test -v -bench=BenchmarkBatchGetBigData -run=^$ -benchtime=10x ./，测试结果如下：

BenchmarkBatchGetBigData-12           10         244921403 ns/op
PASS
ok      go_orm_demo_1/t1        4.354s

可以看到，小JSON的读取明显快很多！

结论

大JSON数据的读写性能低！！！

avatar
文章
阅读
粉丝