RIAK-KV集群部署
安装basho源
curl -s https://packagecloud.io/install/repositories/basho/riak/script.rpm.sh | sudo bash
###
https://packagecloud.io/basho/riak/packages/el/7/riak-2.2.3-1.el7.centos.x86_64.rpm
wget --content-disposition https://packagecloud.io/basho/riak/packages/el/7/riak-2.2.3-1.el7.centos.x86_64.rpm/download.rpm
安装riak-kv在所有的机器上
yum install -y riak.x86_64
系统优化注意
vi /etc/selinux/config
SELINUX=disabled
vi /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
riak-kv配置在所有机器
sed -i "s/storage_backend = bitcask/storage_backend = leveldb/g" /etc/riak/riak.conf
echo "erlang.schedulers.force_wakeup_interval = 500" >>/etc/riak/riak.conf
echo "erlang.schedulers.compaction_of_load = false" >>/etc/riak/riak.conf
echo "ring_size = 256" >>/etc/riak/riak.conf
IP=`ip addr | grep "scope global eth0"|awk '{ print $2; }' | sed 's/\\/.*$//'`
sed -i -e "s/^nodename = .*/nodename = riak@$IP/" -e "s/^listener.http.internal = .*/listener.http.internal = $IP:8098/" -e "s/^listener.protobuf.internal = .*/listener.protobuf.internal = $IP:8087/" /etc/riak/riak.conf
sed -i -e "s#^platform_data_dir = .*#platform_data_dir = /data/riak#" /etc/riak/riak.conf
mkdir /data/riak;chown -R riak:riak /data/riak
#开启web管理面板
sed -i "s/riak_control = off/riak_control = on/" /etc/riak/riak.conf
md5-f4f8a12e419553d6cd63d3222392dbd0
riak start
riak ping
riak version
riak-admin test
chkconfig riak on
md5-c0346b63e5152669feae94b74c530a29
riak-admin cluster join riak@192.168.1.10
md5-9cc6a666f925418b489eb7632e91287a
riak-admin cluster plan
riak-admin cluster commit
riak-admin cluster status
riak-admin status | grep -A10 ring_members
riak-admin cluster partitions
riak-admin status | grep ring
md5-6e09d77f87f664a79a819246cea10310
sed -i "s/^storage_backend = .*/##storage_backend = bitcask/" /etc/riak/riak.conf
echo "buckets.default.allow_mult = true" >>/etc/riak/riak.conf
echo "javascript.map_pool_size = 0" >>/etc/riak/riak.conf
echo "javascript.reduce_pool_size = 0" >>/etc/riak/riak.conf
echo "javascript.hook_pool_size = 0" >>/etc/riak/riak.con
md5-a48283089632c94725cdd6ecad3b7061
vi advanced.config
[
{eleveldb, [
{total_leveldb_mem_percent, 30}
]},
{riak_kv, [
{add_paths, ["/usr/lib64/riak-cs/lib/riak_cs-2.1.1/ebin"]},
{storage_backend, riak_cs_kv_multi_backend},
{multi_backend_prefix_list, [{<<"0b:">>, be_blocks}]},
{multi_backend_default, be_default},
{multi_backend, [
{be_default, riak_kv_eleveldb_backend, [
{data_root, "/var/lib/riak/leveldb"}
]},
{be_blocks, riak_kv_bitcask_backend, [
{data_root, "/var/lib/riak/bitcask"}
]}
]}
]}
].
md5-a48283089632c94725cdd6ecad3b7061
#重启所有的riak-kv
riak restart
md5-f68a4529a8e8c55d3cb10b40c6add961
vi /etc/stanchion/stanchion.conf
listener = 10.200.33.66:8085
ssl.certfile = $(platform_etc_dir)/cert.pem
ssl.keyfile = $(platform_etc_dir)/key.pem
riak_host = 10.200.33.66:8087
##以下的key和secret刚安装时候是没有的，先保持原配置不修改
admin.key = EKXKWCQRR43CDT8MXHUJ
admin.secret = _iSM6mDuTocK4TAGAGzcd2UmCd5hamDCSkG0qw==
platform_bin_dir = /usr/sbin
platform_data_dir = /var/lib/stanchion
platform_etc_dir = /etc/stanchion
platform_lib_dir = /usr/lib64/stanchion/lib
platform_log_dir = /var/log/stanchion
log.console = file
log.console.level = info
log.console.file = $(platform_log_dir)/console.log
log.console.size = 10MB
log.console.rotation = $D0
log.console.rotation.keep = 5
log.error.file = $(platform_log_dir)/error.log
log.error.size = 10MB
log.error.rotation = $D0
log.error.rotation.keep = 5
log.syslog = off
log.crash = on
log.crash.file = $(platform_log_dir)/crash.log
log.crash.maximum_message_size = 64KB
log.crash.size = 10MB
log.crash.rotation = $D0
log.crash.rotation.keep = 5
nodename = stanchion@10.200.33.66
distributed_cookie = riak
erlang.async_threads = 64
erlang.max_ports = 65536
md5-a48283089632c94725cdd6ecad3b7061
chkconfig stanchion on
stanchion start
md5-340345012a8945ec98199cf3760d8a19
vi /etc/riak-cs/advanced.config
[
{riak_cs,
[
{max_buckets_per_user, 1000}
]}
].
vi /etc/riak-cs/riak-cs.conf
listener = 0.0.0.0:8080
riak_host = 10.200.33.66:8087
stanchion_host = 10.200.33.66:8085
stanchion.ssl = on
##先设置on，创建admin的账号，再改成off
anonymous_user_creation = off
##以下的key和secret刚安装时候是没有的，先保持原配置不修改
admin.key = EKXKWCQRR43CDT8MXHUJ
admin.secret = _iSM6mDuTocK4TAGAGzcd2UmCd5hamDCSkG0qw==
root_host = riak-cs.im.server
pool.request.size = 128
pool.request.overflow = 0
pool.list.size = 5
pool.list.overflow = 0
max_buckets_per_user = 100
max_key_length = 1024
trust_x_forwarded_for = off
max_scheduled_delete_manifests = 50
gc.leeway_period = 24h
gc.interval = 15m
gc.retry_interval = 6h
active_delete_threshold = 0
fast_user_get = off
stats.access.flush_factor = 1
stats.access.flush_size = 1000000
stats.access.archive_period = 1h
stats.access.archiver.max_backlog = 2
stats.access.archiver.max_workers = 2
stats.storage.archive_period = 1d
stats.usage_request_limit = 744
server.name = Riak CS
log.access = on
log.access.dir = $(platform_log_dir)
cs_version = 10300
proxy_get = off
dtrace = off
log.console = file
log.console.level = info
log.console.file = $(platform_log_dir)/console.log
log.console.size = 10MB
log.console.rotation = $D0
log.console.rotation.keep = 5
log.error.file = $(platform_log_dir)/error.log
log.error.size = 10MB
log.error.rotation = $D0
log.error.rotation.keep = 5
log.syslog = off
log.crash = on
log.crash.file = $(platform_log_dir)/crash.log
log.crash.maximum_message_size = 64KB
log.crash.size = 10MB
log.crash.rotation = $D0
log.crash.rotation.keep = 5
platform_log_dir = /var/log/riak-cs
nodename = riak-cs@10.200.33.66
distributed_cookie = riak
erlang.async_threads = 64
erlang.max_ports = 65536
md5-a48283089632c94725cdd6ecad3b7061
chkconfig riak-cs on
riak-cs start
md5-9b7dbbb6323373ccd9e3e727a8bc1c43
curl -H 'Content-Type: application/json' \\
-XPOST http://10.200.33.66:8080/riak-cs/user \\
--data '{"email":"liaoxin@blizzmi.com", "name":"admin"}'
md5-86650f192bd7d0eb17b0e5a5ce267350
把生成的key和secret复制到riak-cs和stanchion的配置文件里面， riakcs的anonymous_user_creation = off
"key_id":"EKXKWCQRR43CDT8MXHUJ",
"key_secret":"_iSM6mDuTocK4TAGAGzcd2UmCd5hamDCSkG0qw==",
重启riak-cs和stanchion
md5-1d077735043599d39977fe59f0f80c1c
cat /etc/nginx/conf.d/default.conf
upstream riak_cs_host {
server 10.200.33.66:8080;
server 10.200.33.67:8080;
server 10.200.33.68:8080;
server 10.200.33.69:8080;
server 10.200.33.70:8080;
server 10.200.33.71:8080;
server 10.200.33.72:8080;
server 10.200.33.73:8080;
server 10.200.33.74:8080;
server 10.200.33.75:8080;
}
server {
listen 80;
server_name riak-cs.mi.server *.riak-cs.mi.server;
access_log /var/log/nginx/riak_cs.access.log;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_redirect off;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 64k;
proxy_buffers 8 64k;
proxy_pass http://riak_cs_host;
add_header 'Access-Control-Allow-Origin' '*';
}
}
注意riak-cs.mi.server *.riak-cs.mi.server 需要在dns解析里面做指向nginx
md5-10b10e4606343fd6480dfc2866eaa0bf
下载:s3cmd-1.6.1.tar.gz（http://sourceforge.net/projects/s3tools/files/s3cmd/）
安装:python setup.py install
配置:s3cmd -c ~/.s3cfg --configure
输入key和secret，https：no，proxy：riak-cs.im.server，prot：8080
md5-a48283089632c94725cdd6ecad3b7061
vi .s3cfg
[default]
access_key = EKXKWCQRR43CDT8MXHUJ
access_token =
add_encoding_exts =
add_headers =
bucket_location = US
ca_certs_file =
cache_file =
check_ssl_certificate = True
check_ssl_hostname = True
cloudfront_host = riak-cs.im.server
default_mime_type = binary/octet-stream
delay_updates = False
delete_after = False
delete_after_fetch = False
delete_removed = False
dry_run = False
enable_multipart = True
encoding = UTF-8
encrypt = False
expiry_date =
expiry_days =
expiry_prefix =
follow_symlinks = False
force = False
get_continue = False
gpg_command = /bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase =
guess_mime_type = True
host_base = riak-cs.im.server
host_bucket = %(bucket)s.riak-cs.im.server
human_readable_sizes = False
invalidate_default_index_on_cf = False
invalidate_default_index_root_on_cf = True
invalidate_on_cf = False
kms_key =
limitrate = 0
list_md5 = False
log_target_prefix =
long_listing = False
max_delete = -1
mime_type =
multipart_chunk_size_mb = 15
multipart_max_chunks = 10000
preserve_attrs = True
progress_meter = True
#proxy_host = riak-cs.im.server #如果域名不能做方域名解析，桶的域名则无法解析，就需要配置riakcs的地址端口做代理访问
#proxy_port = 8080
proxy_host =
proxy_port = 0
put_continue = False
recursive = False
recv_chunk = 65536
reduced_redundancy = False
requester_pays = False
restore_days = 1
secret_key = _iSM6mDuTocK4TAGAGzcd2UmCd5hamDCSkG0qw==
send_chunk = 65536
server_side_encryption = False
signature_v2 = True
simpledb_host = sdb.amazonaws.com
skip_existing = False
socket_timeout = 300
stats = False
stop_on_error = False
storage_class =
urlencoding_mode = normal
use_https = False
use_mime_magic = True
verbosity = WARNING
website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
website_error =
website_index = index.html
md5-5d3e04d05a1802e48aa273527cc1dc29
[default]
access_key = ETJRRPQFVYEOFFFV6_H2
access_token =
add_encoding_exts =
add_headers =
bucket_location = US
ca_certs_file =
cache_file =
check_ssl_certificate = True
check_ssl_hostname = True
cloudfront_host = riak-cs1.dev.blizzmi.local
default_mime_type = binary/octet-stream
delay_updates = False
delete_after = False
delete_after_fetch = False
delete_removed = False
dry_run = False
enable_multipart = True
encoding = UTF-8
encrypt = False
expiry_date =
expiry_days =
expiry_prefix =
follow_symlinks = False
force = False
get_continue = False
gpg_command = /usr/bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase = 123456
guess_mime_type = True
host_base = riak-cs1.dev.blizzmi.local
host_bucket = %(bucket)s.riak-cs1.dev.blizzmi.local
human_readable_sizes = False
invalidate_default_index_on_cf = False
invalidate_default_index_root_on_cf = True
invalidate_on_cf = False
kms_key =
limitrate = 0
list_md5 = False
log_target_prefix =
long_listing = False
max_delete = -1
mime_type =
multipart_chunk_size_mb = 15
multipart_max_chunks = 10000
preserve_attrs = True
progress_meter = True
proxy_host = 192.168.200.33
proxy_port = 8080
put_continue = False
recursive = False
recv_chunk = 65536
reduced_redundancy = False
requester_pays = False
restore_days = 1
secret_key = oT8ugahP33FZAXyQguOpEWPcwjddH2yKnPpfKQ==
send_chunk = 65536
server_side_encryption = False
signature_v2 = True
skip_existing = False
socket_timeout = 300
stats = False
stop_on_error = False
storage_class =
urlencoding_mode = normal
use_https = False
use_mime_magic = True
verbosity = WARNING
website_endpoint = http://%(bucket)s.s3-website-%(location)s.riak-cs1.dev.blizzmi.local/
website_error =
website_index = index.html
md5-a48283089632c94725cdd6ecad3b7061
#创建桶
s3cmd mb s3://test-bucket
#查看桶
s3cmd mb ls
s3cmd info s3://test-bucket
#上传文件
dd if=/dev/zero of=test_file bs=1M count=2
s3cmd put test_file s3://test-bucket
s3cmd ls s3://test-bucket
md5-a48283089632c94725cdd6ecad3b7061
web访问权限配置：
vi policy.txt
{
"Version": "2017-09-17",
"Statement": [
{
"Sid": "AddPerm",
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
"Resource":["arn:aws:s3:::test-bucket/*"]
}
]
}
s3cmd setpolicy policy.txt s3://test-bucket
web下载资源：
域名指向单台测试：
wget http://riak-cs.im.server:8080/test-bucket/test_file
wget http://test-bucket.riak-cs.im.server:8080/test_file
域名指向nginx测试：
wget http://riak-cs.im.server/test-bucket/test_file
md5-8bff76d5e65f873dd62bf8fa1663a21e
s3cmd --acl-private setacs3://bl-mdd-file 这个设为私有，
s3cmd --acl-public setacl s3://bl-mdd-file/* 这个设为公共读,这个是把桶里已经存的图片的读权限权开，后面上传的图片不会公共读，需要再次执行。
要想让桶里的所有数据公共可读，而不是在上传的时侯给object加public-read或是需要每次上传后都去执行上面那个命令，需要给桶加一个policy.目前我们外网的mdd的桶policy如下：
参考：
http://docs.basho.com/riak/cs/2.1.1/references/apis/storage/s3/put-bucket-policy/
http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html
1.txt如下：
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "AddPerm",
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:GetObject"],
"Resource":["arn:aws:s3:::bl-mdd-file/*"]
}
]
}
用s3cmd把这个policy加到桶上如下：
s3cmd setpolicy /home/sharp/1.txt s3://bl-mdd-file
查看各类信息
s3cmd info s3://bl-mdd-file
s3cmd info s3://bl-mdd-file/abc_test_123.png
