Encryption Error

Hi …
Please find the code below :

func Storj(ctx context.Context,satelliteAddress string, encryptionPassphrase string, apiKey uplink.APIKey,
bucketName, uploadPath string, dataToUpload byte) error {
//ctx = context.Background()

//upl, err := uplink.NewUplink(ctx, nil)
fmt.Println("Creating New Uplink Object")
var cfg uplink.Config
cfg.Volatile.TLS.SkipPeerCAWhitelist = true
upl, err := uplink.NewUplink(ctx, &cfg)
//upl, err := uplink.NewUplink(ctx, nil)
if err != nil {
    return fmt.Errorf("could not create new Uplink object: %v", err)
}
defer upl.Close()


fmt.Println("Open Project ")
proj, err := upl.OpenProject(ctx, satelliteAddress, apiKey)
if err != nil {
    return fmt.Errorf("could not open project: %v", err)
}
defer proj.Close()



fmt.Println("Create Bucket " ,bucketName)
_, err = proj.CreateBucket(ctx, bucketName, nil)
fmt.Println("Create Bucket " ,err)
if err != nil {
    return fmt.Errorf("could not create bucket: %v", err)
}

fmt.Println("Creating new encryption ",encryptionPassphrase)

fmt.Println("List of bucket ")
list := uplink.BucketListOptions{
	Direction: storj.Forward}
for {
	result, err := proj.ListBuckets(ctx, &list)
	if err != nil {
		return err
	}
	for _, bucket := range result.Items {
		fmt.Println("Bucket: %v\n", bucket.Name)
	}
	if !result.More {
		break
	}
	list = list.NextPage(result)
}


fmt.Println("Phrase ")
encryptionKey, err := proj.SaltedKeyFromPassphrase(ctx, encryptionPassphrase)
fmt.Println("Phra errpr",err)
if err != nil {
    return fmt.Errorf("could not create encryption key: %v", err)
}
access := uplink.NewEncryptionAccessWithDefaultKey(*encryptionKey)

fmt.Println("Opening created bucket",bucketName);
bucket, err := proj.OpenBucket(ctx, bucketName, access)
fmt.Println("Opening bucket",err)
if err != nil {
    return fmt.Errorf("could not open bucket %q: %v", bucketName, err)
}
defer bucket.Close()




fmt.Println("Converting Data to Buffer");
buf := bytes.NewBuffer(dataToUpload)
if err != nil {
	fmt.Println(err)
}


err = bucket.UploadObject(ctx, uploadPath, buf, nil)
fmt.Println("Error uploading object ",err);
if err != nil {
    return fmt.Errorf("could not upload: %v", err)
}

fmt.Println("Opening bucket for downloading");
// Initiate a download of the same object again
readBack, err := bucket.OpenObject(ctx, uploadPath)
if err != nil {
	return fmt.Errorf("could not open object at %q: %v", uploadPath, err)
}
defer readBack.Close()

fmt.Println("Downloading range");
// We want the whole thing, so range from 0 to -1
strm, err := readBack.DownloadRange(ctx, 0, -1)
if err != nil {
	return fmt.Errorf("could not initiate download: %v", err)
}
defer strm.Close()
fmt.Println("Read from the stream");
// Read everything from the stream
receivedContents, err := ioutil.ReadAll(strm)
if err != nil {
	return fmt.Errorf("could not read object: %v", err)
}

return nil

}

Error :
Error for storj test network (storj sim):
1) When i am creating a bucket using above code and then try to upload data in that bucket , data is not getting upload to storj test network (storj-sim) but if i create bucket using Minio web GUI (localhost:11000 ) and then run only data uploading code . Above program uploads data to the bucket but after uploading data bucket becomes inaccessible when i try to check bucket data in web based GUI (localhost:11000) , so when i click on bucket name from the sub menu (localhost:11000) page refresh and opens first bucket in the list

Error on storj network (live) :
1)When i created bucket using above code and trying to upload data then program is displaying error
Error uploading object segement error : ecclient error :
And when i am trying to upload data to the bucket (created by above program) using Uplink cli then error displayed
Fatel error : decryption failed , check encryption key

Question

  1. Is this error is due to encryption key ?
    2)If above error is due to encryption key then how can i manage encryption key error ?
  2. if now how can i solve this issue ?

@littleskunk i had problems to with stefans setellite and encryption, i thougth this were a release problem, but it could be a more common problem.

Hi… @BlackDuck , how you solved this problem ?

Hey @shivam201312, I altered your script to debug this issue on my computer, and once I got it working with my local storj-sim, this is what I had

package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"

	"storj.io/storj/lib/uplink"
	"storj.io/storj/pkg/storj"
)

func main() {
	fmt.Println("hello")
	key, err := uplink.ParseAPIKey("13YqgFXLeVwY5naWMLXFT886fPHzY21jd2CkWSjnGsPx383x5W38YVRDXsif4JWicZxuRe14zAUVi4vUgFKe7NDWtfxioB4YJJP8c9C")
	if err != nil {
		fmt.Println(err)
		return
	}

	err = makebucketanduploadfile(context.Background(), "localhost:10000", "TestEncryptionKey", key, "mybucket", "myfilepath", []byte("datatouploadasdf"))
	if err != nil {
		fmt.Println(err)
	}
}

func makebucketanduploadfile(ctx context.Context, satelliteAddress string, encryptionPassphrase string, apiKey uplink.APIKey,
	bucketName, uploadPath string, dataToUpload []byte) error {
	//ctx = context.Background()

	//upl, err := uplink.NewUplink(ctx, nil)
	fmt.Println("Creating New Uplink Object")
	var cfg uplink.Config
	cfg.Volatile.TLS.SkipPeerCAWhitelist = true
	upl, err := uplink.NewUplink(ctx, &cfg)
	//upl, err := uplink.NewUplink(ctx, nil)
	if err != nil {
		return fmt.Errorf("could not create new Uplink object: %v", err)
	}
	defer upl.Close()

	fmt.Println("Open Project ")
	proj, err := upl.OpenProject(ctx, satelliteAddress, apiKey)
	if err != nil {
		return fmt.Errorf("could not open project: %v", err)
	}
	defer proj.Close()

	fmt.Println("Create Bucket ", bucketName)
	_, err = proj.CreateBucket(ctx, bucketName, nil)
	fmt.Println("Create Bucket ", err)
	if err != nil {
		return fmt.Errorf("could not create bucket: %v", err)
	}

	fmt.Println("Creating new encryption ", encryptionPassphrase)

	fmt.Println("List of bucket ")
	list := uplink.BucketListOptions{
		Direction: storj.Forward,
	}
	for {
		result, err := proj.ListBuckets(ctx, &list)
		if err != nil {
			return err
		}
		for _, bucket := range result.Items {
			fmt.Println("Bucket: %s\n", bucket.Name)
		}
		if !result.More {
			break
		}
		list = list.NextPage(result)
	}

	fmt.Println("Phrase ")
	encryptionKey, err := proj.SaltedKeyFromPassphrase(ctx, encryptionPassphrase)
	fmt.Println("Phra errpr", err)
	if err != nil {
		return fmt.Errorf("could not create encryption key: %v", err)
	}
	access := uplink.NewEncryptionAccessWithDefaultKey(*encryptionKey)

	fmt.Println("Opening created bucket", bucketName)
	bucket, err := proj.OpenBucket(ctx, bucketName, access)
	fmt.Println("Opening bucket", err)
	if err != nil {
		return fmt.Errorf("could not open bucket %q: %v", bucketName, err)
	}
	defer bucket.Close()

	fmt.Println("Converting Data to Buffer")
	buf := bytes.NewBuffer(dataToUpload)
	if err != nil {
		fmt.Println(err)
	}

	err = bucket.UploadObject(ctx, uploadPath, buf, nil)
	fmt.Println("Error uploading object ", err)
	if err != nil {
		return fmt.Errorf("could not upload: %v", err)
	}

	fmt.Println("Opening bucket for downloading")
	// Initiate a download of the same object again
	readBack, err := bucket.OpenObject(ctx, uploadPath)
	if err != nil {
		return fmt.Errorf("could not open object at %q: %v", uploadPath, err)
	}
	defer readBack.Close()

	fmt.Println("Downloading range")
	// We want the whole thing, so range from 0 to -1
	strm, err := readBack.DownloadRange(ctx, 0, -1)
	if err != nil {
		return fmt.Errorf("could not initiate download: %v", err)
	}
	defer strm.Close()
	fmt.Println("Read from the stream")
	// Read everything from the stream
	receivedContents, err := ioutil.ReadAll(strm)
	if err != nil {
		return fmt.Errorf("could not read object: %v", err)
	}

	fmt.Println("received", string(receivedContents))

	return nil
}

Could you try running this and let me know what happens? The API key you use will need to be different, and you also might see an error related to mismatching RS numbers (which can be updated in the setDefaults() function inside lib/uplink/project.go).

I am still seeing the other issue you mentioned, which is that I can see the bucket in minio, but not the file after running the script. I am not sure the root cause of this yet, but I’d like to see what happens when you try out the script I pasted.

2 Likes

@shivam201312
To access file uploaded programmatically with cmd uplink you need to serialize access used with OpenBucket, put it into separate file and add new parameter to config.yaml.

To serialize:
access := uplink.NewEncryptionAccessWithDefaultKey(*encryptionKey) fmt.Println(access.Serialize())

Parameter for config.yaml:
enc.enc-access-filepath: "/home/wywrzal/.local/share/storj/uplink/access.txt"

1 Like

Thank you @michaln and @moby for replying :smiley::smiley:
I will get back to you after making changes .

@michaln i have created file

and provided paramter :
api-key: “apikey”
enc.enc-access-filepath: “/home/xyz/.local/share/storj/uplink/access.txt”

When i run ls command for listing object in bucket output is :

Ok, you listed bucket without a problem. It looks to be empty. Do you expect to have a file there?

@michaln,

Satellite: mars.tardigrade.io:7777

I did the following:

  1. Created a new file access.txt and pasted the serialized access key.
  2. Uncomment and provided the file path:
    enc.enc-access-filepath: “~/.local/share/storj/uplink/access.txt”

Here are my findings:

  1. When I do: ./uplink_linux_amd64 ls

It is not listing my buckets.

  1. When I do: ./uplink_linux_amd64 ls sj://karl-test/test/

It is not listing any objects from the specified bucket.

Snapshot from my terminal:

////////////////////////////////////////////////////////////////////////////////////////

Here is the output, but without the serialized access in config.yaml file.

////////////////////////////////////////////////////////////////////////////////////////

I upload a file from my go code at the spcified uploadPath.

////////////////////////////////////////////////////////////////////////////////////////!

Yes. I do. I uploaded a bson file at the specified upload path.

hmm not sure why you have empty listing. I took your code, modified it a little bit and I was able to use uplink to list my results. Please take a look at this gist - https://gist.github.com/mniewrzal/31cf28f1c5745d0ea6e4840a5dbeb951

Output from this code in my case was:

    Creating New Uplink Object
    Open Project 
    Create Bucket  my-bucket
    Create Bucket  <nil>
    Creating new encryption  abc
    List of bucket 
    Bucket: %v
     my-bucket
    Serialized access:  12Vtx3EyTEX3Xnh...
    Opening created bucket my-bucket
    Opening bucket <nil>
    Converting Data to Buffer
    Error uploading object  <nil>
    Opening bucket for downloading
    Downloading range
    Read from the stream
    Successful upload/download

And my uplink result:
Screenshot%20from%202019-07-25%2011-39-43

2 Likes

@michaln Thank you .

Let me know if this helps you somehow.

@michaln now , i am able to list object from the bucket using uplink cli
Thank you

1 Like