Get best of the week

Templat server backend Golang - bagian 2 (REST API)

Templat server Golang di bawah ini disiapkan untuk mentransfer pengetahuan dalam tim kami. Tujuan utama dari template, selain pelatihan, adalah untuk mengurangi waktu untuk membuat prototipe tugas server kecil di Go.


Bagian pertama dari template dikhususkan untuk server HTTP:


  • Mengkonfigurasi server HTTP melalui baris perintah dan file konfigurasi
  • Konfigurasikan pengaturan server HTTP TLS
  • pengaturan router dan pendaftaran HTTP dan prof-handler
  • mengatur pencatatan lalu lintas HTTP, pencatatan kesalahan HTTP
  • HTTP Basic dan MS AD Authentication, JSON Web Token
  • memulai server dengan harapan kembali ke saluran kesalahan
  • menggunakan konteks untuk menghentikan server dan layanan terkait dengan benar
  • penanganan kesalahan khusus dan pencatatan kustom
  • perakitan kode dengan implementasi versi, tanggal pembuatan dan komit

Bagian kedua dari templat ini dikhususkan untuk membuat prototipe API REST.
Tautan ke repositori proyek tetap sama.


Arsitektur Template API REST


Dalam proses pengujian templat di dudukan , hasil berikut diperoleh .


  • dalam mode membaca langsung dari PostgreSQL - hingga 16.000 [get / sec], concurrency 1024, median 60 [ms]. Setiap Dapatkan data permintaan dari dua tabel dengan ukuran total 360.000.000 baris. Ukuran JSON adalah 1800 byte.
  • 100 000 — 120 000 [get/sec], oncurrency 1024, 2 [ms]. Transfer rate 200-250 [MB/sec]
  • PostgreSQL — 10 000 [insert/sec].



    • ,
    • Marshal / Unmarshal JSON
  3. SQLXX
    • SQL
  4. DB
  5. JSON
    • Marshal / Unmarshal JSON
    • Unmarshal JSON
    • JSON
  6. HTTP
    • GET, POST, PUT


1.


.


  1. Model :
    • Marshal / Unmarshal JSON
    • -
  2. SQLXX :
    • SQL
  3. DB :
    • , SQL
  4. JSON JSON :
    • Marshal / Unmarshal JSON
    • JSON
  5. HTTP HTTP :
    • HTTP
    • HTTP
    • (basic, MS AD, ...)
    • token
    • (body)
    • (header)
    • HSTS Strict-Transport-Security
    • Content-Type (response)
    • HTTP
    • (header)
    • defer recovery panic

REST API : FaaS, , gRPC.


2 Model


2.1. ,


2.1.1


, "Department" "Employee". .
, JOSN .
(-).


// Dept represent object "Department"
type Dept struct {
    Deptno int         `db:"deptno" json:"deptNumber" validate:"required"`
    Dname  string      `db:"dname" json:"deptName" validate:"required"`
    Loc    null.String `db:"loc" json:"deptLocation,nullempty"`
    Emps   []*Emp      `json:"emps,omitempty"`
}

// Emp represent object "Employee"
type Emp struct {
    Empno    int         `db:"empno" json:"empNo" validate:"required"`
    Ename    null.String `db:"ename" json:"empName,nullempty"`
    Job      null.String `db:"job" json:"job,nullempty"`
    Mgr      null.Int    `db:"mgr" json:"mgr,omitempty"`
    Hiredate null.String `db:"hiredate" json:"hiredate,nullempty"`
    Sal      null.Int    `db:"sal" json:"sal,nullempty" validate:"gte=0"`
    Comm     null.Int    `db:"comm" json:"comm,nullempty" validate:"gte=0"`
    Deptno   null.Int    `db:"deptno" json:"deptNumber,nullempty"`
}

.
( ).
, , - (, ). "" .
REST API , : Get, Create, Update.
context.Context — " ".
, , , , , . sync.Pool.


// DeptService represent basic interface for Dept
type DeptService interface {
    GetDept(ctx context.Context, out *Dept) (bool, error)
    GetDeptsPK(ctx context.Context, out *DeptPKs) error
    CreateDept(ctx context.Context, in *Dept, out *Dept) error
    UpdateDept(ctx context.Context, in *Dept, out *Dept)  (bool, error)
}

// EmpService represent basic interface for Emp
type EmpService interface {
    GetEmp(ctx context.Context, out *Emp) (bool, error)
    GetEmpsByDept(ctx context.Context, in *Dept, out *EmpSlice) error
    CreateEmp(ctx context.Context, in *Emp, out *Emp) error
    UpdateEmp(ctx context.Context, in *Emp, out *Emp)  (bool, error)
}

2.1.2


IBM MQ.


// Request represent a type for request
type Request struct {
    QObject               string //  
    QueueName             string //    
    QueueOptions          int32  // MQOO_OUTPUT, MQOO_BROWSE, MQOO_INPUT_AS_Q_DEF  
    QueueWaitInterval     int    //      seconds
    MsgIdHex              string //    Hex 
    CorrelIdHex           string //     Hex 
    MesFormat             string // MQFMT_NONE, MQFMT_STRING
    MesSyncpoint          int32  // MQPMO_NO_SYNCPOINT, MQPMO_SYNCPOINT
    MesOptions            int32  //
    BrowseOptions         int32  // MQGMO_BROWSE_FIRST, MQGMO_BROWSE_NEXT, ...
    Bufsize               int    //  
    Buf                   []byte `json:"-"` //   
}

// Response represent a type for response
type Response struct {
    QObject      string //  
    QueueName    string //    
    MsgId        []byte //    byte 
    MsgIdHex     string //    Hex 
    CorrelId     []byte //     byte 
    CorrelIdHex  string //     Hex 
    PutDate      string //     
    PutTime      string //     
    DataLen      int    //  
    ErrCode      string //  
    ErrMes       string //   
    ErrTrace     string //    
    CauseErrCode string //    - 
    CauseErrMes  string //    - 
    Buf          []byte //   
}

IBM MQ.


// QueueService represent basic service for queue
type QueueService interface {
    OpenQueue(ctx context.Context, req *Request, resp *Response) error
    CloseQueue(ctx context.Context, req *Request, resp *Response) error
    BrowseMessage(ctx context.Context, req *Request, resp *Response) error
    BrowseFirstMessage(ctx context.Context, req *Request, resp *Response) error
    BrowseNextMessage(ctx context.Context, req *Request, resp *Response) error
    BrowseUnderMessage(ctx context.Context, req *Request, resp *Response) error
    GetMessage(ctx context.Context, req *Request, resp *Response) error
    DeleteUnderMessage(ctx context.Context, req *Request, resp *Response) error
    PutMessage(ctx context.Context, req *Request, resp *Response) error
    RegisterCallback(ctx context.Context, req *Request, resp *Response) error
    DeregisterCallback(ctx context.Context, req *Request, resp *Response) error
}

2.2.


, :


  1. — , . , "" > 0, "" not null.
  2. — , " " >= " "
  3. () — , " " >= " "
  4. — , (PK, UK), .
  5. — , (FK), .
  6. — , .

1, 2 3 . 4, 5, 6 , , .


.
, , , .


  • .
  • " API". , DML . , API. , Oracle , API.
  • (View API), , .
  • Instead of Trigger DML API.

, — DML . ( Oracle ). API, .
.


, 1, 2 3 JSON .


3 1, 2 3 gopkg.in/go-playground/validator.
4 (PK, UK).
5 6 .


2.3.


REST API (JSON->Model->DB) (). , , JSON . API, (GC). ( GC).


GC — sync.Pool. . — sync.Pool.
, — sync.Pool, , , , .


sync.Pool :


  • —
  • —

Dept


// deptsPool represent depts pooling
var deptsPool = sync.Pool{
    New: func() interface{} { return new(Dept) },
}

// GetDept allocates a new struct or grabs a cached one
func GetDept() *Dept {
    p := deptsPool.Get().(*Dept)
    p.Reset()
    return p
}

// PutDept return struct to cache
func PutDept(p *Dept, isCascad bool) {
    if p != nil {
        PutEmpSlice(p.Emps, isCascad)
        p.Emps = nil
        deptsPool.Put(p)
    }
}

// Reset reset all fields in structure - use for sync.Pool
func (p *Dept) Reset() {
    p.Deptno = 0
    p.Dname = ""
    p.Loc.String = ""
    p.Loc.Valid = false
    if p.Emps != nil {
        EmpSlice(p.Emps).Reset()
    }
    p.Emps = nil
}



//EmpSlice represent slice of Emps
type EmpSlice []*Emp

// PutEmpSlice return struct to cache
func PutEmpSlice(p EmpSlice) {
    if p != nil {
        for i := range p {
            p[i] = nil //      
        }
        p = p[:0] //   
        empSlicePool.Put(p)
    }
}

, , 1000000 Get , 500 . .
, sync.Pool CPU, sync.Pool .
dept emp , .


2.4. Marshal / Unmarshal JSON


encoding/json .
:



. .
:



github.com/mailru/easyjson.


, , / JSON .


//easyjson:json
type Dept struct {
    Deptno int         `db:"deptno" json:"deptNumber" validate:"required"`
    Dname  string      `db:"dname" json:"deptName" validate:"required"`
    Loc    null.String `db:"loc" json:"deptLocation,nullempty"`
    Emps   []*Emp      `json:"emps,omitempty"`
}

, easyjson NULL , gopkg.in/guregu/null.v4. , NULL , , "Loc null.String" "Loc sql.NullString".


.


easyjson model.go

Marshal / Unmarshal , JSON.


3. SQLXX


SQLXX :


  • , , Listen/notify PostgreSQL (jackc/pgx/stdlib lib/pq)

, .
, , SQL .


httpserver/sqlxx jmoiron/sqlx.


// DB is a wrapper around sqlx.DB
type DB struct {
    *sqlx.DB

    cfg     *Config
    sqlStms SQLStms // SQL 
}

// Tx is an sqlx wrapper around sqlx.Tx
type Tx struct {
    *sqlx.Tx
}

, , jmoiron/sqlx, , jackc/pgx.


3.1.


jmoiron/sqlx , : Beginx, Rollback, Commit, Select, Get, Exec.
.
INFO trace. , , , , .


func Commit(reqID uint64, tx *Tx) (myerr error) {
    //    
    defer func() {
        r := recover()
        if r != nil {
            msg := "PostgreSQL recover from panic: commit transaction: reqID"
            switch t := r.(type) {
            case string:
                myerr = myerror.New("4008", msg, reqID, t).PrintfInfo()
            case error:
                myerr = myerror.WithCause("4006", msg, t, reqID).PrintfInfo()
            default:
                myerr = myerror.New("4006", msg, reqID).PrintfInfo()
            }
        }
    }()

    //     
    if tx == nil {
        return myerror.New("4004", "Transaction is not defined: reqID", reqID).PrintfInfo()
    }

    if err := tx.Commit(); err != nil {
        return myerror.WithCause("4008", "Error commit the transaction: reqID", err, reqID).PrintfInfo()
    }
    return nil
}

interface{}, , , , .


func Select(reqID uint64, tx *Tx, sqlStm *SQLStm, dest interface{}, args ...interface{}) (myerr error) {
    if dest != nil && !reflect.ValueOf(dest).IsNil() {
        // ...
    }
    return myerror.New("4400", "Incorrect call - nil dest interface{} pointer: reqID", reqID).PrintfInfo()
}

reflect.ValueOf(in).IsNil() , .


3.2.


. .


// Config   
type Config struct {
    ConnectString   string //    
    Host            string // host 
    Port            string //   
    Dbname          string //  
    SslMode         string //  SSL
    User            string //     
    Pass            string //  
    ConnMaxLifetime int    //     
    MaxOpenConns    int    //    
    MaxIdleConns    int    //    
    DriverName      string //   "postgres" | "pgx" | "godror"
}

3.3. SQL


SQLXX SQL interface{}.
, SQL .


SQLXX SQL . SQL , , ( SQL ).


// SQLStm represent SQL text and sqlx.Stmt
type SQLStm struct {
    Text      string     //  SQL 
    Stmt      *sqlx.Stmt //  SQL 
    IsPrepare bool       // ,     SQL 
}

// SQLStms represent SQLStm map
type SQLStms map[string]*SQLStm

4. DB


DB :



4.1. , SQL


, (-) :


  • (PK) — API. GUID, , BTree .
  • (UK), , " ". UK .
  • (FK) PK. (FK), FK API . FK.

dept emp, , — .


DB SQL .
SQL : "GetDept", "GetDeptUK", "DeptExists" .. , SQL.


, SQL :


  • PK, , "GetDept"
  • UK, , "GetDeptUK"
  • PK UK, , "DeptExists"

, View API. SQL REST API . .


service.SQLStms = map[string]*sql.SQLStm{
    "GetDept":         &sql.SQLStm{"SELECT deptno, dname, loc FROM dept WHERE deptno = $1", nil, true},
    "GetDeptUK":       &sql.SQLStm{"SELECT deptno, dname, loc FROM dept WHERE deptno = $1", nil, true},
    "DeptExists":      &sql.SQLStm{"SELECT 1 FROM dept WHERE deptno = $1", nil, true},
    "GetDepts":        &sql.SQLStm{"SELECT deptno, dname, loc FROM dept", nil, true},
    "GetDeptsPK":      &sql.SQLStm{"SELECT deptno FROM dept", nil, true},
    "CreateDept":      &sql.SQLStm{"INSERT INTO dept (deptno, dname, loc) VALUES (:deptno, :dname, :loc)", nil, false},
    "UpdateDept":      &sql.SQLStm{"UPDATE dept SET dname = :dname, loc = :loc WHERE deptno = :deptno", nil, false},
    "EmpExists":       &sql.SQLStm{"SELECT 1 FROM emp WHERE empno = $1", nil, true},
    "GetEmp":          &sql.SQLStm{"SELECT empno, ename, job, mgr, hiredate, sal, comm, deptno FROM emp WHERE empno = $1", nil, true},
    "GetEmpUK":        &sql.SQLStm{"SELECT empno, ename, job, mgr, hiredate, sal, comm, deptno FROM emp WHERE empno = $1", nil, true},
    "GetEmpsByDept":   &sql.SQLStm{"SELECT empno, ename, job, mgr, hiredate, sal, comm, deptno FROM emp WHERE deptno = $1", nil, true},
    "GetEmpsPKByDept": &sql.SQLStm{"SELECT empno FROM emp WHERE deptno = $1", nil, true},
    "CreateEmp":       &sql.SQLStm{"INSERT INTO emp (empno, ename, job, mgr, hiredate, sal, comm, deptno) VALUES (:empno, :ename, :job, :mgr, :hiredate, :sal, :comm, :deptno)", nil, false},
    "UpdateEmp":       &sql.SQLStm{"UPDATE emp SET empno = :empno, ename = :ename, job = :job, mgr = :mgr, hiredate = :hiredate, sal = :sal, comm = :comm, deptno = :deptno WHERE empno = :empno", nil, false},
}

, SQL "".


4.2.


:


  • .
  • .

4.2.1 Get


Get . , .


func (s *Service) getDept(ctx context.Context, tx *mysql.Tx, out *model.Dept) (exists bool, myerr error) {

    //   
    if exists, myerr = s.db.Get(reqID, tx, "GetDept", out, out.Deptno); myerr != nil {
        return false, myerr
    }

    //   
    if exists {
        outEmps := model.GetEmpSlice() //   pool    
        if myerr = s.getEmpsByDept(ctx, tx, out, &outEmps); myerr != nil {
            return false, myerr
        }
        out.Emps = outEmps //     
    }

    return exists, nil
}

Get .


func (s *Service) GetDept(ctx context.Context, out *model.Dept) (exists bool, myerr error) {
    return s.getDept(ctx, nil, out)
}

4.2.2 Create


Create :


  • newDept, model.GetDept(). , .
  • . UK ( Dept ).
    • Get(reqID, tx, "DeptExists", &foo, in.Deptno), — UK. args ...interface{}, .
    • , — . .
  • PK
    • . , SQL / 1 . , .
    • API, , / PK, , Get(reqID, tx, "GetDeptUK", newDept, in.Deptno) UK.

    • , FK newEmp.Deptno = null.Int{sql.NullInt64{int64(newDept.Deptno), true}} createEmp(ctx, tx, newEmp, nil).
    • , .
  • — PK
    • — JSON .
    • HTTP .
    • , nil out.

func (s *Service) createDept(ctx context.Context, tx *mysql.Tx, in *model.Dept, out *model.Dept) (myerr error) {

    newDept := model.GetDept()         //   pool      
    defer model.PutDept(newDept, true) //    pool

    { // ,        UK
        var foo int
        exists, myerr := s.db.Get(reqID, tx, "DeptExists", &foo, in.Deptno)
        if myerr != nil {
            return myerr
        }
        if exists {
            return myerror.New("4004", "Error create - row already exists: reqID, Deptno", reqID, in.Deptno).PrintfInfo()
        }
    } // ,        UK

    { //       PK
        rows, myerr := s.db.Exec(reqID, tx, "CreateDept", in)
        if myerr != nil {
            return myerr
        }
        //    
        if rows != 1 {
            return myerror.New("4004", "Error create: reqID, Deptno, rows", reqID, in.Deptno, rows).PrintfInfo()
        }

        //    -     ,   
        //    UK,    PK    
        exists, myerr := s.db.Get(reqID, tx, "GetDeptUK", newDept, in.Deptno)
        if myerr != nil {
            return myerr
        }
        if !exists {
            return myerror.New("4004", "Row does not exists after creating: reqID, Deptno", reqID, in.Deptno).PrintfInfo()
        }
    } //       PK

    { //       
        if in.Emps != nil {
            for _, newEmp := range in.Emps {
                //   PK     
                newEmp.Deptno = null.Int{sql.NullInt64{int64(newDept.Deptno), true}}

                //   
                if myerr = s.createEmp(ctx, tx, newEmp, nil); myerr != nil {
                    return myerr
                }
            }
        }
    } //       

    //     
    if out != nil {
        out.Deptno = newDept.Deptno //    PK
        exists, myerr := s.getDept(ctx, tx, out)
        if myerr != nil {
            return myerr
        }
        //     API
        if !exists {
            return myerror.New("4004", "Row does not exists after updating: reqID, PK", reqID, in.Deptno).PrintfInfo()
        }
    }
    return nil
}

Create :


  • rollback commit

func (s *Service) CreateDept(ctx context.Context, in *model.Dept, out *model.Dept) (myerr error) {
    var tx *mysql.Tx
    reqID := myctx.FromContextRequestID(ctx) // RequestID   context

    //   
    if tx, myerr = s.db.Beginx(reqID); myerr != nil {
        return myerr
    }

    //     
    if myerr = s.createDept(ctx, tx, in, out); myerr != nil {
        _ = s.db.Rollback(reqID, tx)
        return myerr
    }

    //  
    return s.db.Commit(reqID, tx)
}

4.2.3 Update


Update :


  • oldDept newDept, .
  • PK.
    • — . .

    • , . , .
    • UK, UK.

    • . , SQL 1 .
    • API, , / , , P Get(reqID, tx, "GetDept", newDept, in.Deptno).

    • , FK updateEmp(ctx, tx, inEmp, nil).
    • ,
  • — PK

func (s *Service) updateDept(ctx context.Context, tx *mysql.Tx, in *model.Dept, out *model.Dept) (exists bool, myerr error) {

    oldDept := model.GetDept()         //   pool      
    defer model.PutDept(oldDept, true) //    pool

    newDept := model.GetDept()         //   pool      
    defer model.PutDept(newDept, true) //    pool

    { //         
        oldDept.Deptno = in.Deptno //    PK
        if exists, myerr = s.getDept(ctx, tx, oldDept); myerr != nil {
            return false, myerr
        }
        if !exists {
            mylog.PrintfDebugMsg("Row does not exists: reqID, PK", reqID, in.Deptno)
            return false, nil
        }
    } //         

    { //   /        
        //   UK
        //  Dept PK  UK  -     
        if oldDept.Deptno != in.Deptno {
            var foo int
            exists, myerr := s.db.Get(reqID, tx, "DeptExists", &foo, in.Deptno)
            if myerr != nil {
                return false, myerr
            }
            if exists {
                return false, myerror.New("4004", "Error update - row with UK already exists: reqID, Deptno", reqID, in.Deptno).PrintfInfo()
            }
        }
    } //   /        

    { //  
        rows, myerr := s.db.Exec(reqID, tx, "UpdateDept", in)
        if myerr != nil {
            return false, myerr
        }
        //    
        if rows != 1 {
            return false, myerror.New("4004", "Error update: reqID, Deptno, rows", reqID, in.Deptno, rows).PrintfInfo()
        }

        //   -     PK
        exists, myerr := s.db.Get(reqID, tx, "GetDept", newDept, in.Deptno)
        if myerr != nil {
            return false, myerr
        }
        if !exists {
            return false, myerror.New("4004", "Row does not exists after creating: reqID, PK", reqID, in.Deptno).PrintfInfo()
        }
    } //  

    { //       
        if in.Emps != nil {
            for _, inEmp := range in.Emps {
                //   PK     
                inEmp.Deptno = null.Int{sql.NullInt64{int64(in.Deptno), true}}

                //   
                if exists, myerr = s.updateEmp(ctx, tx, inEmp, nil); myerr != nil {
                    return false, myerr
                }
                //        -   
                if !exists {
                    if myerr = s.createEmp(ctx, tx, inEmp, nil); myerr != nil {
                        return false, myerr
                    }
                }
            }
        }
    } //       

    //     
    if out != nil {
        out.Deptno = in.Deptno //    PK
        if exists, myerr = s.getDept(ctx, tx, out); myerr != nil {
            return false, myerr
        }
        //     API
        if !exists {
            return false, myerror.New("4004", "Row does not exists after updating: reqID, PK", reqID, in.Deptno).PrintfInfo()
        }
    }
    return true, nil
}

Update :


  • rollback commit
  • , exists = false

func (s *Service) UpdateDept(ctx context.Context, in *model.Dept, out *model.Dept) (exists bool, myerr error) {
    var tx *mysql.Tx
    reqID := myctx.FromContextRequestID(ctx) // RequestID   context

    //   
    if tx, myerr = s.db.Beginx(reqID); myerr != nil {
        return false, myerr
    }

    //     
    if exists, myerr = s.updateDept(ctx, tx, in, out); myerr != nil {
        _ = s.db.Rollback(reqID, tx)
        return false, myerr
    }

    //            ,  
    if !exists {
        _ = s.db.Rollback(reqID, tx)
        return false, nil
    }

    //  
    if myerr = s.db.Commit(reqID, tx); myerr != nil {
        return false, myerr
    }
    return exists, nil
}

4.3


, .


. , backend , . / , .


3 dgraph-io/ristretto. PostgreSQL listen/notify.


5. JSON


JSON :


  • Marshal / Unmarshal JSON
  • JSON

JSON . - ( ).


JSON HTTP. , . []byte, JSON.


5.1. Marshal JSON


encode JSON github.com/mailru/easyjson .


, encode JSON Dept:


  • HTTP JSON buf []byte
  • , w.Buffer.BuildBytes(buf) ,
  • HTTP

func deptMarshal(reqID uint64, v *model.Dept, buf []byte) (outBuf []byte, myerr error) {
    w := jwriter.Writer{} //  EasyJSON Writer
    v.MarshalEasyJSON(&w) //  JSON    EasyJSON Writer

    if w.Error != nil {
        return nil, myerror.WithCause("6001", "Error Marshal: reqID", w.Error, reqID).PrintfInfo(1)
    }

    //     EasyJSON Writer   
    return w.Buffer.BuildBytes(buf), nil
    }

5.2. Unmarshal JSON


Decode JSON :


if err := vIn.UnmarshalJSON(inBuf); err != nil {
    return 0, nil, myerror.WithCause("6001", "Error Unmarshal: reqID, buf", err, reqID, string(inBuf)).PrintfInfo()
}

. mailru/easyjson decode , . make new, "" .
:


  • out.Emps = make([]*Emp, 0, 8) out.Emps = []*Emp(GetEmpSlice())
  • v1 = new(Emp) v1 = GetEmp()

5.3.


5.3.1. Get


Get :


  • : id — PK, , buf — JSON
  • : outBuf JSON ( buf)
  • JSON . vOut := model.GetDept() ( Dept — , Emp — ) defer model.PutDept(vOut, true). .

func (s *Service) GetDept(ctx context.Context, id int, buf []byte) (outBuf []byte, myerr error) {

    vOut := model.GetDept()         //   pool  
    defer model.PutDept(vOut, true) //   pool     

    vOut.Deptno = id                // PK     

    //   
    exists, myerr := s.deptService.GetDept(ctx, vOut)
    if myerr != nil {
        return nil, myerr
    }

    //  json
    if exists {
        return deptMarshal(reqID, vOut, buf)
    }

    return nil, nil //    - ,    
}

5.3.2. Create


Create :


  • inBuf — JSON
  • id — PK

func (s *Service) CreateDept(ctx context.Context, inBuf []byte, buf []byte) (id int, outBuf []byte, myerr error) {

    vIn := model.GetDept()         //   pool  
    defer model.PutDept(vIn, true) //   pool 

    vOut := model.GetDept()         //   pool  
    defer model.PutDept(vOut, true) //   pool 

    //  JSON  
    if err := vIn.UnmarshalJSON(inBuf); err != nil {
        return 0, nil, myerror.WithCause("6001", "Error Unmarshal: reqID, buf", err, reqID, string(inBuf)).PrintfInfo()
    }

    //   
    if myerr = s.deptService.CreateDept(ctx, vIn, vOut); myerr != nil {
        return 0, nil, myerr
    }

    //  json
    if outBuf, myerr = deptMarshal(reqID, vOut, buf); myerr != nil {
        return 0, nil, myerr
    }

    return vOut.Deptno, outBuf, myerr
}

5.3.3. Update


Update:


  • id — PK .
  • outBuf NO_DATA_FOUND

func (s *Service) UpdateDept(ctx context.Context, id int, inBuf []byte, buf []byte) (outBuf []byte, myerr error) {

    vIn := model.GetDept()         //   pool  
    defer model.PutDept(vIn, true) //   pool 

    vOut := model.GetDept()         //   pool  
    defer model.PutDept(vOut, true) //   pool 

    //  JSON  
    if err := vIn.UnmarshalJSON(inBuf); err != nil {
        return nil, myerror.WithCause("6001", "Error Unmarshal: reqID, buf", err, reqID, string(inBuf)).PrintfInfo()
    }

    //  ID 
    if id != vIn.Deptno {
        return nil, myerror.New("6001", "Resource ID does not corespond to JSON: resource.id, json.Deptno", id, vIn.Deptno).PrintfInfo()
    }

    //   
    exists, myerr := s.deptService.UpdateDept(ctx, vIn, vOut)
    if myerr != nil {
        return nil, myerr
    }

    //  json
    if exists {
        return deptMarshal(reqID, vOut, buf)
    }

    return nil, nil //    - ,    
}

5.4. JSON


:


  • encode JSON
  • .

3 JSON go.etcd.io/bbolt. PostgreSQL listen/notify.


6. HTTP


HTTP :


  • HTTP
  • HTTP
  • (basic, MS AD, ...)
  • token ( )
  • (body)
  • (header)
  • HSTS Strict-Transport-Security
  • Content-Type (response)
  • HTTP
  • (header)
  • defer recovery panic

. HTTP HTTP Golang.


, , :


  • defer . UML — RecoverWrap.func1, .
  • process. HTTP handler. UML process — .
  • HTTP handler. UML EchoHandler.func1 — .

. process, REST API.


( ).


http_handler


6.1.


JSON, []byte http.ResponseWriter . GC. GC .


  • JSON http.ResponseWriter
  • JSON http.ResponseWriter

.


, sync.Pool:



:


[HTTP_POOL]
UseBufPool = true           //    
BufPooledSize = 512         //   ,       
BufPooledMaxSize = 32768    //

process .


var buf []byte
if s.cfg.UseBufPool && s.bytesPool != nil {
    buf = s.bytesPool.GetBuf()
}

, , encode JSON .
, JSON .
[]byte .


if responseBuf != nil && s.cfg.UseBufPool && s.bytesPool != nil {
    //          pool
    if cap(responseBuf) >= s.cfg.BufPooledSize && cap(responseBuf) <= s.cfg.BufPooledMaxSize {
        defer s.bytesPool.PutBuf(responseBuf)
    }
}

, . Pointer() []byte.


if reflect.ValueOf(buf).Pointer() != reflect.ValueOf(responseBuf).Pointer() {
    // ...
}

6.2. GET


process, :


  • URL ID — PK
  • JSON jsonService.GetDept(ctx, id, buf), buf .
  • . , header["RequestID"]

func (s *Service) GetDeptHandler(w http.ResponseWriter, r *http.Request) {
    //   process,   
    _ = s.process("GET", w, r, func(ctx context.Context, requestBuf []byte, buf []byte) ([]byte, Header, int, error) {
        reqID := myctx.FromContextRequestID(ctx) // RequestID   context

        //  PK  URL     
        vars := mux.Vars(r)
        idStr := vars["id"]
        id, err := strconv.Atoi(idStr)
        if err != nil {
            return nil, nil, http.StatusBadRequest, myerror.WithCause("8001", "Failed to process parameter 'id' invalid number: reqID, id", err, reqID, idStr).PrintfInfo()
        }

        //  JSON ,     
        responseBuf, err := s.jsonService.GetDept(ctx, id, buf)
        if err != nil {
            return nil, nil, http.StatusInternalServerError, err
        }

        //    
        if responseBuf == nil {
            return nil, nil, http.StatusNotFound, nil
        }

        //  
        header := Header{}
        header["Content-Type"] = "application/json; charset=utf-8"
        header["Errcode"] = "0"
        header["RequestID"] = fmt.Sprintf("%v", reqID)

        return responseBuf, header, http.StatusOK, nil
    })
}

6.3. POST


POST :


  • ID — PK

func (s *Service) CreateDeptHandler(w http.ResponseWriter, r *http.Request) {
    //   process,   
    _ = s.process("POST", w, r, func(ctx context.Context, requestBuf []byte, buf []byte) ([]byte, Header, int, error) {
        reqID := myctx.FromContextRequestID(ctx) // RequestID   context

        mylog.PrintfDebugMsg("START: reqID", reqID)

        //  JSON 
        id, responseBuf, err := s.jsonService.CreateDept(ctx, requestBuf, buf)
        if err != nil {
            return nil, nil, http.StatusInternalServerError, err
        }

        //  
        header := Header{}
        header["Content-Type"] = "application/json; charset=utf-8"
        header["Errcode"] = "0"
        header["Id"] = fmt.Sprintf("%v", id)
        header["RequestID"] = fmt.Sprintf("%v", reqID)

        return responseBuf, header, http.StatusOK, nil
    })
    }

6.4. PUT


PUT :


func (s *Service) UpdateDeptHandler(w http.ResponseWriter, r *http.Request) {
    //   process,   
    _ = s.process("PUT", w, r, func(ctx context.Context, requestBuf []byte, buf []byte) ([]byte, Header, int, error) {
        reqID := myctx.FromContextRequestID(ctx) // RequestID   context

        mylog.PrintfDebugMsg("START: reqID", reqID)

        //      
        vars := mux.Vars(r)
        idStr := vars["id"]
        id, err := strconv.Atoi(idStr)
        if err != nil {
            return nil, nil, http.StatusBadRequest, myerror.WithCause("8001", "Failed to process parameter 'id' invalid number: reqID, id", err, reqID, idStr).PrintfInfo()
        }

        //  JSON 
        responseBuf, err := s.jsonService.UpdateDept(ctx, id, requestBuf, buf)
        if err != nil {
            return nil, nil, http.StatusInternalServerError, err
        }

        //    
        if responseBuf == nil {
            return nil, nil, http.StatusNotFound, nil
        }

        //  
        header := Header{}
        header["Content-Type"] = "application/json; charset=utf-8"
        header["Errcode"] = "0"
        header["RequestID"] = fmt.Sprintf("%v", reqID)

        return responseBuf, header, http.StatusOK, nil
    })
}

6.5.


, httpserver/httpservice Handlers.


// Handler represent HTTP handler
type Handler struct {
    Path        string
    HundlerFunc func(http.ResponseWriter, *http.Request)
    Method      string
}

// Handlers represent HTTP handlers map
type Handlers map[string]Handler

httpservice.New. , service.recoverWrap().


service.Handlers = map[string]Handler{
    //  
    "EchoHandler":         Handler{"/echo", service.recoverWrap(service.EchoHandler), "POST"},
    "SinginHandler":       Handler{"/signin", service.recoverWrap(service.SinginHandler), "POST"},
    "JWTRefreshHandler":   Handler{"/refresh", service.recoverWrap(service.JWTRefreshHandler), "POST"},
    "HTTPLogHandler":      Handler{"/httplog", service.recoverWrap(service.HTTPLogHandler), "POST"},
    "HTTPErrorLogHandler": Handler{"/httperrlog", service.recoverWrap(service.HTTPErrorLogHandler), "POST"},
    "LogLevelHandler":     Handler{"/loglevel", service.recoverWrap(service.LogLevelHandler), "POST"},

    // JSON 
    "CreateDeptHandler": Handler{"/depts", service.recoverWrap(service.CreateDeptHandler), "POST"},
    "GetDeptHandler":    Handler{"/depts/{id:[0-9]+}", service.recoverWrap(service.GetDeptHandler), "GET"},
    "UpdateDeptHandler": Handler{"/depts/{id:[0-9]+}", service.recoverWrap(service.UpdateDeptHandler), "PUT"},
}

7.


7.1.


Oracle 3- :


  • VM.DenseIO2.8 — 8 Core Intel (16 virtual core), 120 Memory (GB), Oracle Linux 7.7
  • NVMe — 6.4 TB NVMe SSD Storage (1 drives) min 250k IOPS (4k block)
  • — 8.2 Network Bandwidth (Gbps)
  • — PostgreSQL 12. .
  • ApacheBench:
    • concurrency level 4 16384
    • 10 000 1 000 000 .
    • 2 8 ApacheBench ( ApacheBench 1 )

test stand


7.2. GET


  • JSON — 1800 . Dept 10 Emp.
  • Dept — 33 000 000, Emp — 330 000 000 .

:


  • GET PostgreSQL
  • GET JSON BBolt ( 74 )
  • GET ( map)

.


Requests per second Concurrency.


get_db_memory_json1


Grafik ketergantungan waktu per req. [ms] dari Concurrency (skala logaritmik).


get_db_memory_json1


Grafik 99% persentil [ms] versus Concurrency.


get_db_memory_json1


Grafik 99% persentil [ms] versus Concurrency (skala logaritmik).


get_db_memory_json1


Beginilah tampilan server backend pada 100.000 [# / dtk] dan data caching dalam BBolt (Concurrency 128) terlihat seperti. Rata-rata latensi I / O pada drive NVMe adalah 0,02 [ms] (500.000 IOPS).


get_db_memory_json1

More articles:


All Articles