s3cmd 経由による javascript (ajax)で利用できる簡易なs3 web api

が欲しくなったので、メモ。

s3cmd で、テキストやbase64な画像ファイルをput/get な感じです

s3 ...というより IAM( Identity and Access Management ) の設定

以下の3つのポリシーを作成し、利用するユーザ等にアタッチさせます。

1番目の全バケット(*)に対する ListAllMyBuckets は、s3cmdが設定を行う為に必要です。 2,3番目で test-foo-backet に限定してますので、s3cmdが他のbacketに悪影響を及ぼすことはなさそうです。

Effect: Allow
AWS Service: Amazon S3
Actions: ListAllMyBuckets
Amazon Resource Name(ARN): arn:aws:s3:::*

Effect: Allow
AWS Service: Amazon S3
Actions: All Actions Selected
Amazon Resource Name(ARN): arn:aws:s3:::test-foo-backet

Effect: Allow
AWS Service: Amazon S3
Actions: All Actions Selected
Amazon Resource Name(ARN): arn:aws:s3:::test-foo-backet/*

s3cmd の準備 (install & configure)

# yum -y --enablerepo epel install s3cmd
$ s3cmd --configure

「s3cmd --configure」では、Access Key や Secret Key の入力を求められますが、 これらは、AWSマネージメントコンソールの IAM > ユーザ > アクセスキーの管理 で、発行して下さい。(ただ...アクセスキーは、2個/ユーザまでです)

ここから先はsrcを貼ります

development.pl

use File::Spec;
use File::Basename qw(dirname);
my $basedir = File::Spec->rel2abs(File::Spec->catdir(dirname(__FILE__), '..'));
my $dbpath = File::Spec->catfile($basedir, 'db', 'development.db');
+{
  aws_s3 =>
  {content_backet=>'test-backet',
   s3cmd =>'/usr/bin/s3cmd',
   s3cmd_conf =>'/home/endo/.s3cfg',
   backet_cache_dir =>'/home/endo/dev_data/TestAmon2/backet_cache',
  }
};

html template

: cascade "include/Layout.html"

: override head_content -> {
<title>AWS S3 API</title>
: }

: override body_main_content -> {
  <form id="main_form" method="post">
    BASE FOLDER<input type="text" id="base_folder" value="testfolder">
    ユーザID   <input type="text" id="uid"         value="xxx7654321">
    <hr>
    <input type="text" id="form_1" value="HOGE1">
    <input type="text" id="form_2" value="ほげ２">

    <button type="button" onClick="aws_s3.save_txt_data()">UPLOAD TXT</button>
    <button type="button" onClick="aws_s3.load_txt_data()">DOWNLOAD TXT</button>
    <button type="button" onClick="aws_s3.ls_data()"      >LISTING DATA(ls)</button>
    <hr>

    <input type="file" id="file_1" style="display:inline; width:350px;">

    <button type="button" onClick="aws_s3.chk_image()">CHECK IMAGE</button>
    <button type="button" onClick="aws_s3.save_image()">UPLOAD IMAGE</button>
    <button type="button" onClick="aws_s3.load_image()">DOWNLOAD IMAGE</button>
    <table class="table table-bordered">
      <tbody>
    <tr>
      <th style="width:100px;">file名</th>
      <td style="width:150px;"id="file_name"></td>
      <th style="width:100px;">mime type</th>
      <td style="width:150px;" id="file_type"></td>
      <th style="width:150px;">file size (byte)</th>
      <td style="width:150px;" id="file_size"></td>
    </tr>
      </tbody>
    </table>
    <img id="thumb_nail" width="400">
    <div id="img_test"></div>

  </form>
: }


: override body_sub_content -> {
: }

: override foot_content -> {
<script src="/static/js/jquery.cookie.js"></script>
<script src="/static/js/aws_s3_api.js"></script>
<script>
 $(document).ready(function(){
   aws_s3.set_base_folder( $('#base_folder').val() );
 });
</script>
: }

javascript

(function() {
    var AwsS3Api = function(){};
    
    AwsS3Api.prototype = {

        chk_image: function(){
            if(! window.File || ! window.FileReader) {
                alert("このブラウザはFile APIをサポートしていません");
                return;
            }
        // jQueryオブジェクトから、元のhtmlオブジェクトを抽出
            var file = $('#file_1').get(0).files[0];
            if(! file || ! file.type.match(/image/)) {
                alert('画像ファイルを選択下さい');
                return;
            }
            $('#file_name').html( file.name );
            $('#file_type').html( file.type );
            $('#file_size').html( file.size );

            var reader = new FileReader();
            var this_obj = this;
            reader.onload = function(evt) {
                $("#thumb_nail").attr('src',reader.result);
                $("#thumb_nail").css('display','block');
            };
            reader.readAsDataURL(file);

        },
        
        save_image : function (){
            var mime_type = $('#file_type').text().split('/');
            if( mime_type[0] != 'image'){
                alert('画像ファイルを選択下さい');
                return;
            }
            
            var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
            var this_obj = this;
            var uid = $('#uid').val();
            var filename = uid + "_base64_image.txt";
            var path =
                [this.base_folder, this.calc_user_path( uid ), filename].join('/');
            var url = '/awss3/put/' + path;

            data.content = $("#thumb_nail").attr('src');

            $.ajax({url: url,
                    type: 'POST',
                    data: data,
                    success: function(data,txt_status,xhr){
                        if(data.result != 'OK'){
                            alert('処理が失敗しました');
                            return;
                        }
                        alert('処理が完了しました');
                    },
                    error: function(data,txt_status,xhr){
                        alert('処理が失敗しました');
                    }
                   });
        },

        load_image : function (){
            var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
            var this_obj = this;

            var uid = $('#uid').val();
            var filename = uid + "_base64_image.txt";
            var path =
                [this.base_folder, this.calc_user_path( uid ), filename].join('/');
            var url = '/awss3/get/' + path;

            $.ajax({url: url,
                    type: 'POST',
                    data: data,
                    success: function(data,txt_status,xhr){
                        if(data.result != 'OK'){
                            alert('処理が失敗しました');
                            return;
                        }
                        $("#thumb_nail").attr('src', data.content);
                        alert('処理が完了しました');
                    },
                    error: function(data,txt_status,xhr){
                        alert('処理が失敗しました');
                    }
                   });
        },

        set_base_folder: function(base_folder){
            this.base_folder = base_folder;
        },

    //複数ユーザのファイルを1個のdirに登録すると、lsがスゴそうですので...
        calc_user_path: function( uid ){
            //ex. uid = uid123456 + chk digit
            var m = uid.match(/^(uid\d\d\d)\d\d\d/i);
            if( m[1] ){
                return m[1].toLowerCase();
            }
            return 'other';
        },

        save_txt_data : function (){
            var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
            var this_obj = this;

            var uid = $('#uid').val();
            var filename = uid + "_answer.txt";
            var path =
                [this.base_folder, this.calc_user_path( uid ), filename].join('/');
            var url = '/awss3/put/' + path;
            data.content = [ $('#form_1').val().replace(/\t/g, ' '),
                             $('#form_2').val().replace(/\t/g, ' ')].join("\t");
            $.ajax({url: url,
                    type: 'POST',
                    data: data,
                    success: function(data,txt_status,xhr){
                        if(data.result != 'OK'){
                            alert('処理が失敗しました');
                            return;
                        }
                        alert('処理が完了しました');
                    },
                    error: function(data,txt_status,xhr){
                        alert('処理が失敗しました');
                    }
                   });
        },

        load_txt_data : function (){
            var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
            var this_obj = this;

            var uid = $('#uid').val();
            var filename = uid + "_answer.txt";
            var path =
                [this.base_folder, this.calc_user_path( uid ), filename].join('/');
            var url = '/awss3/get/' + path;

            $.ajax({url: url,
                    type: 'POST',
                    data: data,
                    success: function(data,txt_status,xhr){
                        if(data.result != 'OK'){
                            alert('処理が失敗しました');
                            return;
                        }

                        var cols = data.content.split("\t");

                        $('#form_1').val(cols[0]);
                        $('#form_2').val(cols[1]);

                        alert('処理が完了しました');
                    },
                    error: function(data,txt_status,xhr){
                        alert('処理が失敗しました');
                    }
                   });
        },
        
        ls_data : function (){
            var data = { 'XSRF-TOKEN': $.cookie('XSRF-TOKEN') };
            var this_obj = this;

            var uid = $('#uid').val();
            var filename = uid + "_answer.txt";
            var path =
                [this.base_folder, this.calc_user_path( uid ), filename].join('/');
            var url = '/awss3/ls/' + path;

            $.ajax({url: url,
                    type: 'POST',
                    data: data,
                    success: function(data,txt_status,xhr){
                        if(data.result != 'OK'){
                            alert('処理が失敗しました');
                            return;
                        }

                        alert( data.content );
                    },
                    error: function(data,txt_status,xhr){
                        alert('処理が失敗しました');
                    }
                   });
        }
    };
    window.aws_s3 = new AwsS3Api();
})();

Controller (perl)

package TestAmon2::Web::Dispatcher::AwsS3Api;
use strict;
use utf8;
use base qw/TestAmon2::Web::Dispatcher/;
use Data::Dumper;       # for debug write
use TestAmon2::Model;
use TestAmon2::Web::View;


sub new {
    my ($class) = @_;
    my $self = {};
    $self =  bless $self, $class;
    return $self;
}

sub do_main {
    my ($self,$c) = @_;

    $self->{c} = $c;
    $self->{user_id} = $self->get_login_user_id($c);
    my $path_info_str = $c->req()->{env}->{PATH_INFO};

    unless( $path_info_str =~ m{^/awss3/([^/]+)/(.*)}o ){
        return $self->do_index_page($c);
    }
    my $action =  $1;
    my $s3_path = $2;

    if($action eq 'ls' or
       $action eq 'get' or
       $action eq 'put'
      ){
        my $method = "do_".$action;
        return $self->$method($c, $s3_path);
    }

    return $self->do_index_page($c);
}

sub do_index_page {
    my ($self, $c) = @_;

    my $render_data = {};
    $render_data->{c} = $c;
    my $view = TestAmon2::Web::View::AwsS3Api->new();

    return $view->render_index_page($c, $render_data);
}

sub do_put {
    my ($self, $c,$s3_path) = @_;

    my $render_data = {};
    $render_data->{c} = $c;
    my $view = TestAmon2::Web::View::AwsS3Api->new();

    my $m_aws_s3 = TestAmon2::Model::AwsS3->new($c);

    my $result = $m_aws_s3->cmd_put($s3_path,
                                    $c->req->param('content'));
    if ($result){
        return $c->render_json( {result=>'OK'});
    }
    return $c->render_json( {result=>'NG'});
}

sub do_get {
    my ($self, $c,$s3_path) = @_;

    my $render_data = {};
    $render_data->{c} = $c;
    my $view = TestAmon2::Web::View::AwsS3Api->new();

    my $m_aws_s3 = TestAmon2::Model::AwsS3->new($c);
    $render_data->{content} = $m_aws_s3->cmd_get($s3_path);

    if(not defined($render_data->{content})){
        return $c->render_json( {result=>'NG'});
    }
    return $c->render_json({result=>'OK',
                            content=> $render_data->{content}});
}

sub do_ls {
    my ($self, $c,$s3_path) = @_;

    my $render_data = {};
    $render_data->{c} = $c;
    my $view = TestAmon2::Web::View::AwsS3Api->new();

    my $m_aws_s3 = TestAmon2::Model::AwsS3->new($c);
    $render_data->{content} = $m_aws_s3->cmd_ls($s3_path);

    if(not defined($render_data->{content})){
        return $c->render_json( {result=>'NG'});
    }
    return $c->render_json({result=>'OK',
                            content=> $render_data->{content}});
}

1;
__END__

Model (perl)

s3に対するput/getは遅いので、ec2側にもcacheしています

package TestAmon2::Model::AwsS3;
use strict;
use utf8;
use base qw/TestAmon2/;
use Cwd;
use File::Path qw/mkpath rmtree/;
use File::Temp;
use MIME::Base64;
use Data::Dumper;       # for debug write

my $CONF = TestAmon2->config;
my $S3CMD = $CONF->{aws_s3}->{s3cmd};
my $S3CMD_CONF = $CONF->{aws_s3}->{s3cmd_conf};
my $BACKET = $CONF->{aws_s3}->{content_backet};
my $CACHE_LIMIT_AGE = 60 * 5; #sec


# http://*.com/awss3/get $BACKET/$FOLDER/$FILE
# s3cmd ls s3://バケット名
# s3cmd put オブジェクト名 s3://バケット名/
# s3cmd get オブジェクト名 s3://バケット名/
# s3cmd del オブジェクト名 s3://バケット名/

sub new {
    my ($class, $c) = @_;

    my $self = {c=>$c};
    $self =  bless $self, $class;
    return $self;
}

sub open_localfile {
    my ($self, $path ) = @_;

    my $fh;
    unless( open($fh, '<', $path) ){
        $self->log_error("fail open $path");
        return;
    }

    local $/ = undef;
    my $content = <$fh>;

    unless( close($fh) ){
        $self->log_error("fail close $path");
        return;
    }
    
    if($path =~ /\.txt$/io ){
        $content = Encode::decode('utf8',$content);
    } else {
        $content = MIME::Base64::encode_base64($content);
    }
    
    return $content;
}


sub cmd_put {
    my ($self, $path, $content ) = @_;

    unless( $path =~ m|^(.+)/([^/]+)$|o){
        $self->log_error("fail parse dir/filename from $path");
        return;
    }
    my $parent_dir = $1;
    my $filename =   $2;

    my $cache_dir = join('/',$CONF->{aws_s3}->{backet_cache_dir},$parent_dir);
    my $cache_path = "$cache_dir/$filename";

    if(not -d $cache_dir){
        unless( File::Path::mkpath($cache_dir)) {
            $self->log_error("fail mkpath $cache_dir");
            return;
        }
    }

    my $fh;
    unless( open($fh, '>', $cache_path) ){
        $self->log_error("fail open $cache_path");
        return;
    }
    print $fh Encode::encode('utf8', $content);

    unless( close($fh) ){
        $self->log_error("fail close $cache_path");
        return;
    }

    my $cmd = join(' ',
                   "$S3CMD --quiet --force --config=$S3CMD_CONF",
                   "put $cache_path s3://$BACKET/$parent_dir/");
    unless( open($fh, '-|', $cmd) ){
        $self->log_error("fail open $cmd");
        return;
    }

    unless( close($fh) ){
        $self->log_error("fail close $cmd");
        return;
    }

    return $path;
}

sub cmd_get {
    my ($self, $path ) = @_;

    unless( $path =~ m|^(.+)/([^/]+)$|o){
        $self->log_error("fail parse dir/filename from $path");
        return;
    }
    my $parent_dir = $1;
    my $filename =   $2;

    my $cache_dir = join('/',$CONF->{aws_s3}->{backet_cache_dir},$parent_dir);
    my $cache_path = "$cache_dir/$filename";

    if(-e $cache_path ){
        my $now =   time();
        my $mtime = (stat($cache_path))[9];
        if(($now - $mtime) < $CACHE_LIMIT_AGE ){
            return $self->open_localfile($cache_path);
        }
    }

    if(not -d $cache_dir){
        unless( File::Path::mkpath($cache_dir)) {
            $self->log_error("fail mkpath $cache_dir");
            return;
        }
    }

    my $cmd = join(' ',
                   "$S3CMD --quiet --force --config=$S3CMD_CONF",
                   "get s3://$BACKET/$path $cache_dir/");
    my $fh;
    unless( open($fh, '-|', $cmd) ){
        $self->log_error("fail open $cmd");
        return;
    }

    unless( close($fh) ){
        $self->log_error("fail close $cmd");
        return;
    }

    unless(utime(undef,undef,$cache_path)){
        $self->log_error("fail utime(touch command) $cache_path");
        return;
    }

    return $self->open_localfile($cache_path);
}

sub cmd_ls {
    my ($self, $path ) = @_;

    my $cmd = join(' ',"$S3CMD --config=$S3CMD_CONF","ls s3://$BACKET/$path");
    my $fh;
    unless( open($fh, '-|', $cmd) ){
        $self->log_error("fail open $cmd");
        return [];
    }

    my @ret;
    while(my $line = <$fh>){
        chomp($line);
        my @cols = split(/\s\s+/, $line);

        $cols[2] =~ s|s3://$BACKET/||o;
        if( $cols[1] eq 'DIR'){
            push(@ret,[$cols[2]]);
        } else {
            push(@ret,[$cols[2],$cols[1],$cols[0]]);
        }
    }
    
    unless( close($fh) ){
        $self->log_error("fail close $cmd");
        return [];
    }
    return \@ret;
}

1;
__END__

View (perl)

package TestAmon2::Web::View::AwsS3Api;
use strict;
use utf8;
use base qw(TestAmon2::Web::View);
use Data::Dumper;       # for debug write

sub new {
    my ($class) = @_;
    my $self = {};
    $self =  bless $self, $class;
    return $self;
}

sub render_ls {
    my ($self, $c, $render_data) = @_;
    return $c->render_json( {body=>$render_data->{paths}});
}

sub render_get {
    my ($self, $c, $render_data) = @_;
    return $c->render_json( {body=>$render_data->{content}});
}

sub render_index_page {
    my ($self, $c, $render_data) = @_;
    return $c->render('AwsS3Api/Index.html', $render_data);
}

1;
__END__