property-system-gateway/utils/minio.go

package utils

import (
	"context"
	"errors"
	"fmt"
	"git.getensh.com/common/gopkgs/cache"
	"github.com/minio/minio-go/v6"
	"io"
	"log"
	"math/rand"
	hurl "net/url"
	"property-system-gateway/parser"
	"property-system-gateway/pb"
	pb_v1 "property-system-gateway/pb/v1"
	"strconv"
	"strings"
	"time"
)

const (
	NUmStr  = "0123456789"
	CharStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	SpecStr = "+=-@#~,.[]()!%^*$"
)

func GenerateRandomStr(length int, charset string) string {
	time.Sleep(1 * time.Microsecond)
	rand.Seed(time.Now().UnixNano())
	//初始化密码切片
	var passwd []byte = make([]byte, length, length)
	//源字符串
	var sourceStr string
	//判断字符类型,如果是数字
	if charset == "num" {
		sourceStr = NUmStr
		//如果选的是字符
	} else if charset == "char" {
		sourceStr = charset
		//如果选的是混合模式
	} else if charset == "mix" {
		sourceStr = fmt.Sprintf("%s%s", NUmStr, CharStr)
		//如果选的是高级模式
	} else if charset == "advance" {
		sourceStr = fmt.Sprintf("%s%s%s", NUmStr, CharStr, SpecStr)
	} else {
		sourceStr = fmt.Sprintf("%s%s%s", NUmStr, CharStr, SpecStr)
	}
	//遍历，生成一个随机index索引,
	for i := 0; i < length; i++ {
		index := rand.Intn(len(sourceStr))
		passwd[i] = sourceStr[index]
	}
	return string(passwd)
}

func UploadToMinioNew(fileName string, r io.Reader, size int64, imgMine string, bucketName string, taskNeed bool) (objName string, err error) {
	endpoint := parser.Conf.Oss.Endpoint
	accessKeyID := parser.Conf.Oss.Id
	secretAccessKey := parser.Conf.Oss.Key
	useSSL := false

	// Initialize minio client object.
	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	if err != nil {
		return "", err
	}

	// Make a new bucket called mymusic.

	contentType := imgMine

	objectName := ""

	count := 0
	exist := false
	array := strings.Split(fileName, ".")
	tail := ""
	if len(array) > 0 {
		tail = array[len(array)-1]
	}
	rkeyPrefix := "oss_obj_exist_"
	rkey := ""
	defer func() {
		if rkey != "" {
			cache.RedisUnlock(rkey)
		}
	}()
	for ; count < 10; count++ {
		if rkey != "" {
			cache.RedisUnlock(rkey)
			rkey = ""
		}
		objectName = fmt.Sprintf("%d####", time.Now().Unix()) + GenerateRandomStr(32, "mix") + "." + tail

		// 互斥判断文件是否存在
		rkey = rkeyPrefix + objectName
		if !cache.RedisLock(rkey) {
			time.Sleep(200 * time.Millisecond)
			rkey = ""
			continue
		}

		exist, err = ObjExist(endpoint, bucketName, objectName)
		if err != nil {
			return "", err
		}
		if exist {
			continue
		}

		// Upload the zip file with FPutObject
		_, err = minioClient.PutObject(bucketName, objectName, r, size, minio.PutObjectOptions{ContentType: contentType})
		if err != nil {
			return "", err
		}
		ret := parser.Conf.Oss.Protocol + "://" + endpoint + "/" + bucketName + "/" + hurl.QueryEscape(objectName)
		if taskNeed {
			ObjTaskAdd(ret)
		}

		return ret, nil
	}

	if exist {
		return "", errors.New("文件已存在")
	}

	return "", errors.New("系统繁忙")
}

func UploadToMinio(fileName string, r io.Reader, size int64, imgMine string) (objName string, err error) {
	endpoint := parser.Conf.Oss.Endpoint
	accessKeyID := parser.Conf.Oss.Id
	secretAccessKey := parser.Conf.Oss.Key
	useSSL := false

	// Initialize minio client object.
	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	if err != nil {
		return "", err
	}

	// Make a new bucket called mymusic.
	bucketName := parser.Conf.Oss.PropertyCompanyBucket

	contentType := imgMine

	objectName := ""

	count := 0
	exist := false
	array := strings.Split(fileName, ".")
	tail := ""
	if len(array) > 0 {
		tail = array[len(array)-1]
	}
	rkeyPrefix := "oss_obj_exist_"
	rkey := ""
	defer func() {
		if rkey != "" {
			cache.RedisUnlock(rkey)
		}
	}()
	for ; count < 10; count++ {
		if rkey != "" {
			cache.RedisUnlock(rkey)
			rkey = ""
		}
		objectName = fmt.Sprintf("%d####", time.Now().Unix()) + GenerateRandomStr(32, "mix") + "." + tail

		// 互斥判断文件是否存在
		rkey = rkeyPrefix + objectName
		if !cache.RedisLock(rkey) {
			time.Sleep(200 * time.Millisecond)
			rkey = ""
			continue
		}

		exist, err = ObjExist(endpoint, bucketName, objectName)
		if err != nil {
			return "", err
		}
		if exist {
			continue
		}

		// Upload the zip file with FPutObject
		_, err = minioClient.PutObject(bucketName, objectName, r, size, minio.PutObjectOptions{ContentType: contentType})
		if err != nil {
			return "", err
		}
		ret := parser.Conf.Oss.Protocol + "://" + endpoint + "/" + bucketName + "/" + hurl.QueryEscape(objectName)
		ObjTaskAdd(ret)
		return ret, nil
	}

	if exist {
		return "", errors.New("文件已存在")
	}

	return "", errors.New("系统繁忙")
}

func parseObjUrl(objUrl string) (string, string, string, string) {
	array := strings.Split(objUrl, "://")
	protocol, endpoint, bucketName, objName := "", "", "", ""
	if len(array) < 2 {
		return "", "", "", ""
	}
	protocol = array[0]
	array = strings.Split(array[1], "/")
	if len(array) != 3 {
		return "", "", "", ""
	}
	endpoint = array[0]
	bucketName = array[1]
	objName = array[2]
	return protocol, endpoint, bucketName, objName
}

const ObjKey = "minio_obj"

func ObjTaskAdd(objUrl string) {
	value := fmt.Sprintf("%s", objUrl)
	cache.Redis().SAdd(ObjKey, value)
}

func delNotExist(objUrl string) error {
	mreq := pb_v1.OssObjDelNotExistRequest{ObjUrl: objUrl}
	_, err := pb.Common.OssObjDelNotExist(context.Background(), &mreq)
	fmt.Printf("************:%v,%v\n", objUrl, err)
	return err
}

func ObjTaskHandle() {
	count := 0
	for count < 20 {
		count++

		objStr, _ := cache.Redis().SPop(ObjKey)
		fmt.Printf("************:%s\n", objStr)
		if objStr == "" {
			break
		}
		_, _, _, objName := parseObjUrl(objStr)
		array := strings.Split(objName, "%23%23%23%23")
		if len(array) != 2 {
			continue
		}
		timeStr := array[0]
		timestamp, _ := strconv.ParseInt(timeStr, 10, 64)
		if false {
			if delNotExist(objStr) != nil {
				cache.Redis().SAdd(ObjKey, objStr)
			}
			continue
		}
		if time.Now().Unix()-timestamp < 3600 {
			cache.Redis().SAdd(ObjKey, objStr)
			time.Sleep(1 * time.Second)
			continue
		}
		delNotExist(objStr)

	}
}

func ObjTask(ctx context.Context) {
	if false {
		tmp := ""
		fmt.Printf("input:\n")
		fmt.Scanln(&tmp)
		fmt.Printf("after input\n")
		ObjTaskHandle()
	}

	t := time.NewTicker(600 * time.Second)
	for {
		select {
		case <-t.C:
			ObjTaskHandle()
		case <-ctx.Done():
			return
		}
	}
}

func CheckObjsExist(objs []string) ([]string, error) {
	ret := []string{}
	for _, v := range objs {
		_, endpoint, bucketName, objName := parseObjUrl(v)
		exist, err := ObjExist(endpoint, bucketName, objName)
		if err != nil {
			return nil, err
		}
		if !exist {
			ret = append(ret, v)
		}
	}
	return ret, nil
}

func ObjExist(endpoint, bucketName, objName string) (bool, error) {
	if endpoint == "" {
		return false, nil
	}
	//endpoint := parser.Conf.Oss.Endpoint
	accessKeyID := parser.Conf.Oss.Id
	secretAccessKey := parser.Conf.Oss.Key
	useSSL := false

	// Initialize minio client object.
	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	if err != nil {
		return false, err
	}

	// Make a new bucket called mymusic.
	//bucketName := parser.Conf.Oss.PropertyCompanyBucket
	//objName := ""
	obj, err := minioClient.GetObject(bucketName, objName, minio.GetObjectOptions{})
	if err != nil {
		return false, err
	}
	defer obj.Close()
	_, err = obj.Stat()
	if err != nil {
		if strings.Contains(err.Error(), " not exist") {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func RemoveFromMinio(objUrl string) (err error) {
	_, endpoint, bucketName, objName := parseObjUrl(objUrl)
	if endpoint == "" {
		return nil
	}
	//endpoint := parser.Conf.Oss.Endpoint
	accessKeyID := parser.Conf.Oss.Id
	secretAccessKey := parser.Conf.Oss.Key
	useSSL := false

	// Initialize minio client object.
	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	if err != nil {
		return err
	}

	// Make a new bucket called mymusic.
	//bucketName := parser.Conf.Oss.PropertyCompanyBucket
	//objName := ""
	err = minioClient.RemoveObject(bucketName, objName)

	if err != nil {
		return err
	}
	return nil
}

func GetFilePath(objName string) (string, error) {
	endpoint := parser.Conf.Oss.Endpoint
	accessKeyID := parser.Conf.Oss.Id
	secretAccessKey := parser.Conf.Oss.Key
	useSSL := false

	// Initialize minio client object.
	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	if err != nil {
		return "", err
	}

	// Make a new bucket called mymusic.
	bucketName := parser.Conf.Oss.PropertyCompanyBucket
	rr, er := minioClient.PresignedGetObject(bucketName, objName, 24*time.Hour, hurl.Values{})
	if er != nil {
		fmt.Printf("获取文件路径失败:%v\n", er)
		return "", er
	}
	return rr.String(), nil
}

func MiniTest() {
	endpoint := "47.108.135.38:9000"
	accessKeyID := "minioadmin"
	secretAccessKey := "hly@1353406"
	useSSL := false

	// Initialize minio client object.
	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	if err != nil {
		log.Fatalln(err)
	}

	// Make a new bucket called mymusic.
	bucketName := "testb"
	location := ""

	err = minioClient.MakeBucket(bucketName, location)
	if err != nil {
		// Check to see if we already own this bucket (which happens if you run this twice)
		exists, errBucketExists := minioClient.BucketExists(bucketName)
		if errBucketExists == nil && exists {
			log.Printf("We already own %s\n", bucketName)
		} else {
			log.Fatalln(err)
		}
	} else {
		log.Printf("Successfully created %s\n", bucketName)
	}

	// Upload the zip file
	objectName := "5.png"
	filePath := "D:\\5.png"
	contentType := ""

	// Upload the zip file with FPutObject

	n, err := minioClient.FPutObject(bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType})
	if err != nil {
		log.Fatalln(err)
	}

	log.Printf("Successfully uploaded %s of size %d\n", objectName, n)
	rr, er := minioClient.PresignedGetObject("testb", objectName, 24*365*100*time.Hour, hurl.Values{})
	if er != nil {
		fmt.Printf("xxxx:%v\n", er)
		return
	}

	fmt.Printf("xxxx:%s\n", rr.String())
}