が欲しくなったので、メモ。
s3cmd で、テキストやbase64な画像ファイルをput/get な感じです
s3 ...というより IAM( Identity and Access Management ) の設定
以下の3つのポリシーを作成し、利用するユーザ等にアタッチさせます。
1番目の全バケット(*)に対する ListAllMyBuckets は、s3cmdが設定を行う為に必要です。
2,3番目で test-foo-backet に限定してますので、s3cmdが他のbacketに悪影響を及ぼすことはなさそうです。
Effect: Allow
AWS Service: Amazon S3
Actions: ListAllMyBuckets
Amazon Resource Name(ARN): arn:aws:s3:::*
Effect: Allow
AWS Service: Amazon S3
Actions: All Actions Selected
Amazon Resource Name(ARN): arn:aws:s3:::test-foo-backet
Effect: Allow
AWS Service: Amazon S3
Actions: All Actions Selected
Amazon Resource Name(ARN): arn:aws:s3:::test-foo-backet/*
s3cmd の準備 (install & configure)
# yum -y --enablerepo epel install s3cmd
$ s3cmd --configure
「s3cmd --configure」では、Access Key や Secret Key の入力を求められますが、
これらは、AWSマネージメントコンソールの IAM > ユーザ > アクセスキーの管理 で、発行して下さい。(ただ...アクセスキーは、2個/ユーザまでです)
ここから先はsrcを貼ります
development.pl
use File::Spec;
use File::Basename qw(dirname);
my $basedir = File::Spec->rel2abs(File::Spec->catdir(dirname(__FILE__), '..'));
my $dbpath = File::Spec->catfile($basedir, 'db', 'development.db');
+{
aws_s3 =>
{content_backet=>'test-backet',
s3cmd =>'/usr/bin/s3cmd',
s3cmd_conf =>'/home/endo/.s3cfg',
backet_cache_dir =>'/home/endo/dev_data/TestAmon2/backet_cache',
}
};
html template
: cascade "include/Layout.html"
: override head_content -> {
<title>AWS S3 API</title>
: }
: override body_main_content -> {
<form id="main_form" method="post">
BASE FOLDER<input type="text" id="base_folder" value="testfolder">
ユーザID <input type="text" id="uid" value="xxx7654321">
<hr>
<input type="text" id="form_1" value="HOGE1">
<input type="text" id="form_2" value="ほげ２">
<button type="button" onClick="aws_s3.save_txt_data()">UPLOAD TXT</button>
<button type="button" onClick="aws_s3.load_txt_data()">DOWNLOAD TXT</button>
<button type="button" onClick="aws_s3.ls_data()" >LISTING DATA(ls)</button>
<hr>
<input type="file" id="file_1" style="display:inline; width:350px;">
<button type="button" onClick="aws_s3.chk_image()">CHECK IMAGE</button>
<button type="button" onClick="aws_s3.save_image()">UPLOAD IMAGE</button>
<button type="button" onClick="aws_s3.load_image()">DOWNLOAD IMAGE</button>
<table class="table table-bordered">
<tbody>
<tr>
<th style="width:100px;">file名</th>
<td style="width:150px;"id="file_name"></td>
<th style="width:100px;">mime type</th>
<td style="width:150px;" id="file_type"></td>
<th style="width:150px;">file size (byte)</th>
<td style="width:150px;" id="file_size"></td>
</tr>
</tbody>
</table>
<img id="thumb_nail" width="400">
<div id="img_test"></div>
</form>
: }
: override body_sub_content -> {
: }
: override foot_content -> {
<script src="/static/js/jquery.cookie.js"></script>
<script src="/static/js/aws_s3_api.js"></script>
<script>
$(document).ready(function(){
aws_s3.set_base_folder( $('#base_folder').val() );
});
</script>
: }
(function() {
var AwsS3Api = function(){};
AwsS3Api.prototype = {
chk_image: function(){
if(! window.File || ! window.FileReader) {
alert("このブラウザはFile APIをサポートしていません");
return;
}
var file = $('#file_1').get(0).files[0];
if(! file || ! file.type.match(/image/)) {
alert('画像ファイルを選択下さい');
return;
}
$('#file_name').html( file.name );
$('#file_type').html( file.type );
$('#file_size').html( file.size );
var reader = new FileReader();
var this_obj = this;
reader.onload = function(evt) {
$("#thumb_nail").attr('src',reader.result);
$("#thumb_nail").css('display','block');
};
reader.readAsDataURL(file);
},
save_image : function (){
var mime_type = $('#file_type').text().split('/');
if( mime_type[0] != 'image'){
alert('画像ファイルを選択下さい');
return;
}
var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
var this_obj = this;
var uid = $('#uid').val();
var filename = uid + "_base64_image.txt";
var path =
[this.base_folder, this.calc_user_path( uid ), filename].join('/');
var url = '/awss3/put/' + path;
data.content = $("#thumb_nail").attr('src');
$.ajax({url: url,
type: 'POST',
data: data,
success: function(data,txt_status,xhr){
if(data.result != 'OK'){
alert('処理が失敗しました');
return;
}
alert('処理が完了しました');
},
error: function(data,txt_status,xhr){
alert('処理が失敗しました');
}
});
},
load_image : function (){
var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
var this_obj = this;
var uid = $('#uid').val();
var filename = uid + "_base64_image.txt";
var path =
[this.base_folder, this.calc_user_path( uid ), filename].join('/');
var url = '/awss3/get/' + path;
$.ajax({url: url,
type: 'POST',
data: data,
success: function(data,txt_status,xhr){
if(data.result != 'OK'){
alert('処理が失敗しました');
return;
}
$("#thumb_nail").attr('src', data.content);
alert('処理が完了しました');
},
error: function(data,txt_status,xhr){
alert('処理が失敗しました');
}
});
},
set_base_folder: function(base_folder){
this.base_folder = base_folder;
},
calc_user_path: function( uid ){
var m = uid.match(/^(uid\d\d\d)\d\d\d/i);
if( m[1] ){
return m[1].toLowerCase();
}
return 'other';
},
save_txt_data : function (){
var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
var this_obj = this;
var uid = $('#uid').val();
var filename = uid + "_answer.txt";
var path =
[this.base_folder, this.calc_user_path( uid ), filename].join('/');
var url = '/awss3/put/' + path;
data.content = [ $('#form_1').val().replace(/\t/g, ' '),
$('#form_2').val().replace(/\t/g, ' ')].join("\t");
$.ajax({url: url,
type: 'POST',
data: data,
success: function(data,txt_status,xhr){
if(data.result != 'OK'){
alert('処理が失敗しました');
return;
}
alert('処理が完了しました');
},
error: function(data,txt_status,xhr){
alert('処理が失敗しました');
}
});
},
load_txt_data : function (){
var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
var this_obj = this;
var uid = $('#uid').val();
var filename = uid + "_answer.txt";
var path =
[this.base_folder, this.calc_user_path( uid ), filename].join('/');
var url = '/awss3/get/' + path;
$.ajax({url: url,
type: 'POST',
data: data,
success: function(data,txt_status,xhr){
if(data.result != 'OK'){
alert('処理が失敗しました');
return;
}
var cols = data.content.split("\t");
$('#form_1').val(cols[0]);
$('#form_2').val(cols[1]);
alert('処理が完了しました');
},
error: function(data,txt_status,xhr){
alert('処理が失敗しました');
}
});
},
ls_data : function (){
var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
var this_obj = this;
var uid = $('#uid').val();
var filename = uid + "_answer.txt";
var path =
[this.base_folder, this.calc_user_path( uid ), filename].join('/');
var url = '/awss3/ls/' + path;
$.ajax({url: url,
type: 'POST',
data: data,
success: function(data,txt_status,xhr){
if(data.result != 'OK'){
alert('処理が失敗しました');
return;
}
alert( data.content );
},
error: function(data,txt_status,xhr){
alert('処理が失敗しました');
}
});
}
};
window.aws_s3 = new AwsS3Api();
})();
Controller (perl)
package TestAmon2::Web::Dispatcher::AwsS3Api;
use strict;
use utf8;
use base qw/TestAmon2::Web::Dispatcher/;
use Data::Dumper;
use TestAmon2::Model;
use TestAmon2::Web::View;
sub new {
my ($class) = @_;
my $self = {};
$self = bless $self, $class;
return $self;
}
sub do_main {
my ($self,$c) = @_;
$self->{c} = $c;
$self->{user_id} = $self->get_login_user_id($c);
my $path_info_str = $c->req()->{env}->{PATH_INFO};
unless( $path_info_str =~ m{^/awss3/([^/]+)/(.*)}o ){
return $self->do_index_page($c);
}
my $action = $1;
my $s3_path = $2;
if($action eq 'ls' or
$action eq 'get' or
$action eq 'put'
){
my $method = "do_".$action;
return $self->$method($c, $s3_path);
}
return $self->do_index_page($c);
}
sub do_index_page {
my ($self, $c) = @_;
my $render_data = {};
$render_data->{c} = $c;
my $view = TestAmon2::Web::View::AwsS3Api->new();
return $view->render_index_page($c, $render_data);
}
sub do_put {
my ($self, $c,$s3_path) = @_;
my $render_data = {};
$render_data->{c} = $c;
my $view = TestAmon2::Web::View::AwsS3Api->new();
my $m_aws_s3 = TestAmon2::Model::AwsS3->new($c);
my $result = $m_aws_s3->cmd_put($s3_path,
$c->req->param('content'));
if ($result){
return $c->render_json( {result=>'OK'});
}
return $c->render_json( {result=>'NG'});
}
sub do_get {
my ($self, $c,$s3_path) = @_;
my $render_data = {};
$render_data->{c} = $c;
my $view = TestAmon2::Web::View::AwsS3Api->new();
my $m_aws_s3 = TestAmon2::Model::AwsS3->new($c);
$render_data->{content} = $m_aws_s3->cmd_get($s3_path);
if(not defined($render_data->{content})){
return $c->render_json( {result=>'NG'});
}
return $c->render_json({result=>'OK',
content=> $render_data->{content}});
}
sub do_ls {
my ($self, $c,$s3_path) = @_;
my $render_data = {};
$render_data->{c} = $c;
my $view = TestAmon2::Web::View::AwsS3Api->new();
my $m_aws_s3 = TestAmon2::Model::AwsS3->new($c);
$render_data->{content} = $m_aws_s3->cmd_ls($s3_path);
if(not defined($render_data->{content})){
return $c->render_json( {result=>'NG'});
}
return $c->render_json({result=>'OK',
content=> $render_data->{content}});
}
1;
s3に対するput/getは遅いので、ec2側にもcacheしています
package TestAmon2::Model::AwsS3;
use strict;
use utf8;
use base qw/TestAmon2/;
use Cwd;
use File::Path qw/mkpath rmtree/;
use File::Temp;
use MIME::Base64;
use Data::Dumper;
my $CONF = TestAmon2->config;
my $S3CMD = $CONF->{aws_s3}->{s3cmd};
my $S3CMD_CONF = $CONF->{aws_s3}->{s3cmd_conf};
my $BACKET = $CONF->{aws_s3}->{content_backet};
my $CACHE_LIMIT_AGE = 60 * 5;
sub new {
my ($class, $c) = @_;
my $self = {c=>$c};
$self = bless $self, $class;
return $self;
}
sub open_localfile {
my ($self, $path ) = @_;
my $fh;
unless( open($fh, '<', $path) ){
$self->log_error("fail open $path");
return;
}
local $/ = undef;
my $content = <$fh>;
unless( close($fh) ){
$self->log_error("fail close $path");
return;
}
if($path =~ /\.txt$/io ){
$content = Encode::decode('utf8',$content);
} else {
$content = MIME::Base64::encode_base64($content);
}
return $content;
}
sub cmd_put {
my ($self, $path, $content ) = @_;
unless( $path =~ m|^(.+)/([^/]+)$|o){
$self->log_error("fail parse dir/filename from $path");
return;
}
my $parent_dir = $1;
my $filename = $2;
my $cache_dir = join('/',$CONF->{aws_s3}->{backet_cache_dir},$parent_dir);
my $cache_path = "$cache_dir/$filename";
if(not -d $cache_dir){
unless( File::Path::mkpath($cache_dir)) {
$self->log_error("fail mkpath $cache_dir");
return;
}
}
my $fh;
unless( open($fh, '>', $cache_path) ){
$self->log_error("fail open $cache_path");
return;
}
print $fh Encode::encode('utf8', $content);
unless( close($fh) ){
$self->log_error("fail close $cache_path");
return;
}
my $cmd = join(' ',
"$S3CMD --quiet --force --config=$S3CMD_CONF",
"put $cache_path s3://$BACKET/$parent_dir/");
unless( open($fh, '-|', $cmd) ){
$self->log_error("fail open $cmd");
return;
}
unless( close($fh) ){
$self->log_error("fail close $cmd");
return;
}
return $path;
}
sub cmd_get {
my ($self, $path ) = @_;
unless( $path =~ m|^(.+)/([^/]+)$|o){
$self->log_error("fail parse dir/filename from $path");
return;
}
my $parent_dir = $1;
my $filename = $2;
my $cache_dir = join('/',$CONF->{aws_s3}->{backet_cache_dir},$parent_dir);
my $cache_path = "$cache_dir/$filename";
if(-e $cache_path ){
my $now = time();
my $mtime = (stat($cache_path))[9];
if(($now - $mtime) < $CACHE_LIMIT_AGE ){
return $self->open_localfile($cache_path);
}
}
if(not -d $cache_dir){
unless( File::Path::mkpath($cache_dir)) {
$self->log_error("fail mkpath $cache_dir");
return;
}
}
my $cmd = join(' ',
"$S3CMD --quiet --force --config=$S3CMD_CONF",
"get s3://$BACKET/$path $cache_dir/");
my $fh;
unless( open($fh, '-|', $cmd) ){
$self->log_error("fail open $cmd");
return;
}
unless( close($fh) ){
$self->log_error("fail close $cmd");
return;
}
unless(utime(undef,undef,$cache_path)){
$self->log_error("fail utime(touch command) $cache_path");
return;
}
return $self->open_localfile($cache_path);
}
sub cmd_ls {
my ($self, $path ) = @_;
my $cmd = join(' ',"$S3CMD --config=$S3CMD_CONF","ls s3://$BACKET/$path");
my $fh;
unless( open($fh, '-|', $cmd) ){
$self->log_error("fail open $cmd");
return [];
}
my @ret;
while(my $line = <$fh>){
chomp($line);
my @cols = split(/\s\s+/, $line);
$cols[2] =~ s|s3://$BACKET/||o;
if( $cols[1] eq 'DIR'){
push(@ret,[$cols[2]]);
} else {
push(@ret,[$cols[2],$cols[1],$cols[0]]);
}
}
unless( close($fh) ){
$self->log_error("fail close $cmd");
return [];
}
return \@ret;
}
1;
package TestAmon2::Web::View::AwsS3Api;
use strict;
use utf8;
use base qw(TestAmon2::Web::View);
use Data::Dumper;
sub new {
my ($class) = @_;
my $self = {};
$self = bless $self, $class;
return $self;
}
sub render_ls {
my ($self, $c, $render_data) = @_;
return $c->render_json( {body=>$render_data->{paths}});
}
sub render_get {
my ($self, $c, $render_data) = @_;
return $c->render_json( {body=>$render_data->{content}});
}
sub render_index_page {
my ($self, $c, $render_data) = @_;
return $c->render('AwsS3Api/Index.html', $render_data);
}
1;
