00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023 #ifdef DRIZZLED
00024 #include <config.h>
00025 #include <drizzled/common.h>
00026 #include <drizzled/session.h>
00027 #include <drizzled/table.h>
00028 #include <drizzled/message/table.pb.h>
00029 #include <drizzled/charset_info.h>
00030 #include <drizzled/table_proto.h>
00031 #include <drizzled/session.h>
00032 #include <drizzled/field.h>
00033 #endif
00034
00035 #include "cslib/CSConfig.h"
00036
00037 #include <stdlib.h>
00038 #include <ctype.h>
00039 #include <string.h>
00040
00041 #include "cslib/CSGlobal.h"
00042 #include "cslib/CSThread.h"
00043 #include "cslib/CSLog.h"
00044 #include "cslib/CSPath.h"
00045 #include "cslib/CSFile.h"
00046 #include "cslib/CSString.h"
00047 #include "cslib/CSStrUtil.h"
00048 #include "cslib/CSStorage.h"
00049 #include "cslib/CSEncode.h"
00050 #include "cslib/CSS3Protocol.h"
00051
00052 #include "backup_ms.h"
00053 #include "cloud_ms.h"
00054
00055 CSSyncSparseArray *MSCloudInfo::gCloudInfo;
00056 uint32_t MSCloudInfo::gMaxInfoRef;
00057
00058 uint32_t CloudDB::gKeyIndex;
00059 CSMutex CloudDB::gCloudKeyLock;
00060
00061
00062 MSCloudInfo::MSCloudInfo(uint32_t id,
00063 const char *server,
00064 const char *bucket_arg,
00065 const char *publicKey,
00066 const char *privateKey
00067 ):
00068 cloudRefId(id),
00069 bucket(NULL),
00070 s3Prot(NULL)
00071 {
00072 new_(s3Prot, CSS3Protocol());
00073 s3Prot->s3_setServer(server);
00074 s3Prot->s3_setPublicKey(publicKey);
00075 s3Prot->s3_setPrivateKey(privateKey);
00076
00077 bucket = CSString::newString(bucket_arg);
00078 }
00079
00080
00081 MSCloudInfo::~MSCloudInfo()
00082 {
00083 if (bucket)
00084 bucket->release();
00085
00086 if (s3Prot)
00087 s3Prot->release();
00088 }
00089
00090
00091 const char *MSCloudInfo::getServer()
00092 {
00093 return s3Prot->s3_getServer();
00094 }
00095
00096
00097 const char *MSCloudInfo::getBucket()
00098 {
00099 return bucket->getCString();
00100 }
00101
00102
00103 const char *MSCloudInfo::getPublicKey()
00104 {
00105 return s3Prot->s3_getPublicKey();
00106 }
00107
00108
00109 const char *MSCloudInfo::getPrivateKey()
00110 {
00111 return s3Prot->s3_getPrivateKey();
00112 }
00113
00114
00115 CSString *MSCloudInfo::getSignature(const char *key, const char *content_type, uint32_t *s3AuthorizationTime)
00116 {
00117 return s3Prot->s3_getAuthorization(bucket->getCString(), key, content_type, s3AuthorizationTime);
00118 }
00119
00120
00121 CSString *MSCloudInfo::getDataURL(const char *key, int keep_alive)
00122 {
00123 return s3Prot->s3_getDataURL(bucket->getCString(), key, keep_alive);
00124 }
00125
00126
00127 void MSCloudInfo::send(CSInputStream *input, const char *key, off64_t size)
00128 {
00129 CSVector *headers;
00130 headers = s3Prot->s3_send(input, bucket->getCString(), key, size);
00131 headers->release();
00132 }
00133
00134
00135 CSVector *MSCloudInfo::list(const char *key_prefix, uint32_t max)
00136 {
00137 return s3Prot->s3_list(bucket->getCString(), key_prefix, max);
00138 }
00139
00140
00141 void MSCloudInfo::receive(CSOutputStream *output, const char *key)
00142 {
00143 bool found;
00144 CSVector *headers;
00145
00146 headers = s3Prot->s3_receive(output, bucket->getCString(), key, &found);
00147 headers->release();
00148 if (!found) {
00149 CSStringBuffer *err;
00150 enter_();
00151
00152 new_(err, CSStringBuffer());
00153 push_(err);
00154 err->append("S3 object not found: ");
00155 err->append(getServer());
00156 err->append("/");
00157 err->append(bucket->getCString());
00158 err->append("/");
00159 err->append(key);
00160
00161 CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, err->getCString());
00162 release_(err);
00163 outer_();
00164 }
00165 }
00166
00167
00168 void MSCloudInfo::cDelete(const char *key)
00169 {
00170 s3Prot->s3_delete(bucket->getCString(), key);
00171 }
00172
00173
00174 void MSCloudInfo::copy(MSCloudInfo *dst_cloud, const char *dst_key, const char *src_key)
00175 {
00176 enter_();
00177 push_(dst_cloud);
00178
00179 s3Prot->s3_copy(dst_cloud->getServer() ,dst_cloud->bucket->getCString(), dst_key, bucket->getCString(), src_key);
00180
00181 release_(dst_cloud);
00182 exit_();
00183 }
00184
00185
00186 CloudDB::CloudDB(uint32_t db_id):
00187 dfltCloudRefId(0),
00188 keep_alive(5 * 60),
00189 blob_recovery_no(0),
00190 blob_db_id(db_id),
00191 isBackup(false),
00192 backupInfo(NULL),
00193 backupCloud(NULL),
00194 clObjectKey(NULL)
00195 {
00196 enter_();
00197
00198 new_(clObjectKey, CSStringBuffer());
00199 clObjectKey->setLength(base_key_size);
00200
00201 exit_();
00202 }
00203
00204
00205 CloudDB::~CloudDB()
00206 {
00207
00208 if (backupInfo)
00209 backupInfo->release();
00210
00211 if (backupCloud)
00212 backupCloud->release();
00213
00214 if (clObjectKey)
00215 clObjectKey->release();
00216
00217 }
00218
00219 MSBackupInfo *CloudDB::cl_getBackupInfo()
00220 {
00221 if (backupInfo)
00222 backupInfo->retain();
00223
00224 return backupInfo;
00225 }
00226
00227
00228 void CloudDB::cl_clearBackupInfo(){ backupInfo->release(); backupInfo = NULL;}
00229
00230
00231 void CloudDB::cl_createDB()
00232 {
00233
00234 }
00235
00236
00237
00238 void CloudDB::cl_restoreDB()
00239 {
00240 CSVector *list = NULL;
00241 CSString *key = NULL;
00242 CloudObjectKey *src_objectKey = NULL, *dst_objectKey = NULL;
00243 CloudKeyRec cloudKey;
00244 uint32_t src_cloudRef, dst_cloudRef = 0;
00245 MSBackupInfo *backup_info = NULL;
00246 MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
00247 enter_();
00248
00249 if (!blob_recovery_no)
00250 exit_();
00251
00252 backup_info = MSBackupInfo::getBackupInfo(blob_recovery_no);
00253 push_(backup_info);
00254
00255 src_cloudRef = backup_info->getcloudRef();
00256 src_cloud = MSCloudInfo::getCloudInfo(src_cloudRef);
00257 push_(src_cloud);
00258
00259 new_(dst_objectKey, CloudObjectKey(blob_db_id));
00260 push_(dst_objectKey);
00261
00262
00263 new_(src_objectKey, CloudObjectKey(blob_db_id));
00264 push_(src_objectKey);
00265 src_objectKey->setObjectKey(NULL, backup_info->getcloudBackupNo(), backup_info->getDatabaseId());
00266
00267
00268 list = src_cloud->list(src_objectKey->getCString());
00269 release_(src_objectKey);
00270 push_(list);
00271
00272
00273
00274 dst_cloudRef = src_cloudRef;
00275 dst_cloud = src_cloud;
00276 dst_cloud->retain();
00277
00278 push_ref_(dst_cloud);
00279
00280 while ((key = (CSString*)(list->take(0))) ) {
00281 push_(key);
00282
00283
00284
00285
00286 CloudObjectKey::parseObjectKey(key->getCString(), &cloudKey);
00287
00288
00289 if (cloudKey.cloud_ref != dst_cloudRef) {
00290 if (dst_cloud) {
00291 dst_cloud->release();
00292 dst_cloud = NULL;
00293 }
00294 dst_cloudRef = cloudKey.cloud_ref;
00295 dst_cloud = MSCloudInfo::getCloudInfo(dst_cloudRef);
00296 }
00297
00298
00299 dst_objectKey->setObjectKey(&cloudKey);
00300 src_cloud->copy(RETAIN(dst_cloud), dst_objectKey->getCString(), key->getCString());
00301 release_(key);
00302
00303 }
00304
00305 release_(dst_cloud);
00306
00307 blob_recovery_no = 0;
00308 release_(list);
00309 release_(dst_objectKey);
00310 release_(src_cloud);
00311 release_(backup_info);
00312 exit_();
00313 }
00314
00315
00316 uint32_t CloudDB::cl_getNextBackupNumber(uint32_t cloud_ref)
00317 {
00318 CloudObjectKey *objectKey;
00319 CSVector *list;
00320 uint32_t backup_no = 0, size = 1;
00321 MSCloudInfo *s3Cloud;
00322 enter_();
00323
00324 s3Cloud = MSCloudInfo::getCloudInfo((cloud_ref)?cloud_ref:dfltCloudRefId);
00325 push_(s3Cloud);
00326
00327 new_(objectKey, CloudObjectKey(blob_db_id));
00328 push_(objectKey);
00329
00330
00331 while (size) {
00332 backup_no++;
00333 objectKey->setObjectKey(NULL, backup_no);
00334 list = s3Cloud->list(objectKey->getCString(), 1);
00335 size = list->size();
00336 list->release();
00337 }
00338
00339 release_(objectKey);
00340 release_(s3Cloud);
00341
00342 return_(backup_no);
00343 }
00344
00345
00346 void CloudDB::cl_backupBLOB(CloudKeyPtr key)
00347 {
00348 CloudObjectKey *src_objectKey, *dst_objectKey;
00349 uint32_t cloudRef, backupNo;
00350 MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
00351 enter_();
00352
00353 ASSERT(backupInfo);
00354
00355 if ((cloudRef = backupInfo->getcloudRef()) == 0) {
00356 backupInfo->setcloudRef(dfltCloudRefId);
00357 cloudRef = dfltCloudRefId;
00358 }
00359
00360 if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {
00361 backupNo = cl_getNextBackupNumber(cloudRef);
00362 backupInfo->setcloudBackupNo(backupNo);
00363 }
00364
00365
00366 new_(src_objectKey, CloudObjectKey(blob_db_id));
00367 push_(src_objectKey);
00368 src_objectKey->setObjectKey(key);
00369
00370
00371 new_(dst_objectKey, CloudObjectKey(blob_db_id));
00372 push_(dst_objectKey);
00373 dst_objectKey->setObjectKey(key, backupNo);
00374
00375
00376 src_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
00377 push_(src_cloud);
00378
00379
00380 dst_cloud = MSCloudInfo::getCloudInfo(cloudRef);
00381 src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
00382
00383 release_(src_cloud);
00384 release_(dst_objectKey);
00385 release_(src_objectKey);
00386 exit_();
00387 }
00388
00389
00390 void CloudDB::cl_restoreBLOB(CloudKeyPtr key, uint32_t backup_db_id)
00391 {
00392 CloudObjectKey *src_objectKey, *dst_objectKey;
00393 uint32_t cloudRef, backupNo;
00394 MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
00395 enter_();
00396
00397 ASSERT(backupInfo);
00398
00399 if ((cloudRef = backupInfo->getcloudRef()) == 0) {
00400 backupInfo->setcloudRef(dfltCloudRefId);
00401 cloudRef = dfltCloudRefId;
00402 }
00403
00404 if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {
00405 backupNo = cl_getNextBackupNumber(cloudRef);
00406 backupInfo->setcloudBackupNo(backupNo);
00407 }
00408
00409
00410 new_(src_objectKey, CloudObjectKey(backup_db_id));
00411 push_(src_objectKey);
00412 src_objectKey->setObjectKey(key, backupNo);
00413
00414
00415 new_(dst_objectKey, CloudObjectKey(blob_db_id));
00416 push_(dst_objectKey);
00417 dst_objectKey->setObjectKey(key);
00418
00419
00420 src_cloud = MSCloudInfo::getCloudInfo(cloudRef);
00421 push_(src_cloud);
00422
00423
00424 dst_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
00425 src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
00426
00427 release_(src_cloud);
00428 release_(dst_objectKey);
00429 release_(src_objectKey);
00430 exit_();
00431 }
00432
00433
00434
00435 void CloudDB::cl_dropDB()
00436 {
00437 CSVector *list;
00438 CSString *key;
00439 CloudObjectKey *objectKey;
00440 MSCloudInfo *s3Cloud = NULL;
00441 int i;
00442 const char *key_str;
00443
00444 enter_();
00445 new_(objectKey, CloudObjectKey(blob_db_id));
00446 push_(objectKey);
00447
00448 lock_(MSCloudInfo::gCloudInfo);
00449
00450 if (isBackup) {
00451 uint32_t backup_no;
00452 if (backupInfo && (backup_no = backupInfo->getcloudBackupNo())) {
00453 objectKey->setObjectKey(NULL, backup_no);
00454 if ((s3Cloud = MSCloudInfo::getCloudInfo(backupInfo->getcloudRef())))
00455 push_(s3Cloud);
00456 }
00457 } else {
00458 objectKey->setObjectKey();
00459 i = 0;
00460 s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++);
00461 }
00462
00463 key_str = objectKey->getCString();
00464
00465
00466
00467
00468 while (s3Cloud) {
00469 list = s3Cloud->list(key_str);
00470 push_(list);
00471
00472
00473 while ((key = (CSString*)(list->take(0))) ) {
00474 push_(key);
00475 s3Cloud->cDelete(key->getCString());
00476 release_(key);
00477 }
00478
00479 release_(list);
00480 if (isBackup) {
00481 release_(s3Cloud);
00482 s3Cloud = NULL;
00483 } else
00484 s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++);
00485 }
00486
00487 unlock_(MSCloudInfo::gCloudInfo);
00488 release_(objectKey);
00489 exit_();
00490 }
00491
00492
00493 void CloudDB::cl_putData(CloudKeyPtr key, CSInputStream *stream, off64_t size)
00494 {
00495 CloudObjectKey *objectKey;
00496 MSCloudInfo *s3Cloud;
00497
00498 enter_();
00499
00500 push_(stream);
00501
00502 new_(objectKey, CloudObjectKey(blob_db_id));
00503 push_(objectKey);
00504
00505 objectKey->setObjectKey(key);
00506
00507 s3Cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
00508 push_(s3Cloud);
00509 s3Cloud->send(RETAIN(stream), objectKey->getCString(), size);
00510 release_(s3Cloud);
00511
00512 release_(objectKey);
00513 release_(stream);
00514
00515 exit_();
00516 }
00517
00518
00519 off64_t CloudDB::cl_getData(CloudKeyPtr key, char *buffer, off64_t size)
00520 {
00521 CloudObjectKey *objectKey;
00522 CSStaticMemoryOutputStream *output;
00523 MSCloudInfo *s3Cloud;
00524 enter_();
00525
00526 new_(objectKey, CloudObjectKey(blob_db_id));
00527 push_(objectKey);
00528
00529 s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
00530 push_(s3Cloud);
00531
00532 new_(output, CSStaticMemoryOutputStream((u_char *)buffer, size));
00533 push_(output);
00534
00535 objectKey->setObjectKey(key);
00536
00537 s3Cloud->receive(RETAIN(output), objectKey->getCString());
00538 size = output->getSize();
00539 release_(output);
00540
00541 release_(s3Cloud);
00542 release_(objectKey);
00543 return_(size);
00544 }
00545
00546
00547 void CloudDB::cl_deleteData(CloudKeyPtr key)
00548 {
00549 MSCloudInfo *s3Cloud;
00550 CloudObjectKey *objectKey;
00551 enter_();
00552
00553 new_(objectKey, CloudObjectKey(blob_db_id));
00554 push_(objectKey);
00555
00556 s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
00557 push_(s3Cloud);
00558
00559 objectKey->setObjectKey(key);
00560
00561 s3Cloud->cDelete(objectKey->getCString());
00562
00563 release_(s3Cloud);
00564 release_(objectKey);
00565
00566 exit_();
00567 }
00568
00569
00570 CSString *CloudDB::cl_getDataURL(CloudKeyPtr key)
00571 {
00572 CloudObjectKey *objectKey;
00573 CSString *url;
00574 MSCloudInfo *s3Cloud;
00575 enter_();
00576
00577 new_(objectKey, CloudObjectKey(blob_db_id));
00578 push_(objectKey);
00579
00580 objectKey->setObjectKey(key);
00581
00582 s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
00583 push_(s3Cloud);
00584
00585 url = s3Cloud->getDataURL(objectKey->getCString(), keep_alive);
00586
00587 release_(s3Cloud);
00588 release_(objectKey);
00589
00590 return_(url);
00591 }
00592
00593
00594 CSString *CloudDB::cl_getSignature(CloudKeyPtr key, CSString *content_type_arg, uint32_t *s3AuthorizationTime)
00595 {
00596 CSString *signature;
00597 CloudObjectKey *objectKey;
00598 const char *content_type = NULL;
00599 MSCloudInfo *s3Cloud;
00600 enter_();
00601
00602 new_(objectKey, CloudObjectKey(blob_db_id));
00603 push_(objectKey);
00604
00605 if (content_type_arg) {
00606 push_(content_type_arg);
00607 content_type = content_type_arg->getCString();
00608 }
00609
00610 objectKey->setObjectKey(key);
00611 s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
00612 push_(s3Cloud);
00613
00614 signature = s3Cloud->getSignature(objectKey->getCString(), content_type, s3AuthorizationTime);
00615
00616 if (content_type_arg)
00617 release_(content_type_arg);
00618
00619 release_(s3Cloud);
00620 release_(objectKey);
00621
00622 return_(signature);
00623 }
00624
00625
00626