From: Christopher Allan Webber Date: Wed, 26 Nov 2014 22:44:28 +0000 (-0600) Subject: This import_export stuff is mongodb days leftovers. Removing! X-Git-Url: https://vcs.fsf.org/?a=commitdiff_plain;h=692e8e78486bf47bb6d83e28d0928c16b175715a;p=mediagoblin.git This import_export stuff is mongodb days leftovers. Removing! This commit sponsored by Ray Redd. Thanks! --- diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index d8d982b1..e5f9a395 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -81,20 +81,6 @@ SUBCOMMAND_MAP = { # 'func': 'mediagoblin.gmg_commands.theme:theme', # 'help': 'Theming commands', # } - - ## These might be useful, mayyyybe, but don't really work anymore - ## due to mongo change and the "versatility" of sql options. - ## - ## For now, commenting out. Might re-enable soonish? - # - # 'env_export': { - # 'setup': 'mediagoblin.gmg_commands.import_export:import_export_parse_setup', - # 'func': 'mediagoblin.gmg_commands.import_export:env_export', - # 'help': 'Exports the data for this MediaGoblin instance'}, - # 'env_import': { - # 'setup': 'mediagoblin.gmg_commands.import_export:import_export_parse_setup', - # 'func': 'mediagoblin.gmg_commands.import_export:env_import', - # 'help': 'Imports the data for this MediaGoblin instance'}, } diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py deleted file mode 100644 index fbac09f6..00000000 --- a/mediagoblin/gmg_commands/import_export.py +++ /dev/null @@ -1,256 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from mediagoblin import mg_globals -from mediagoblin.db.open import setup_connection_and_db_from_config -from mediagoblin.gmg_commands import util as commands_util -from mediagoblin.storage.filestorage import BasicFileStorage -from mediagoblin.init import setup_storage, setup_global_and_app_config - -import shutil -import tarfile -import tempfile -import subprocess -import os.path -import os -import sys -import logging -from contextlib import closing - -_log = logging.getLogger('gmg.import_export') -logging.basicConfig() -_log.setLevel(logging.INFO) - - -def import_export_parse_setup(subparser): - # TODO: Add default - subparser.add_argument( - 'tar_file') - subparser.add_argument( - '--mongodump_path', default='mongodump', - help='mongodump binary') - subparser.add_argument( - '--mongorestore_path', default='mongorestore', - help='mongorestore binary') - subparser.add_argument( - '--cache_path', - help='Temporary directory where files will be temporarily dumped') - - -def _import_media(db, args): - ''' - Import media files - - Must be called after _import_database() - ''' - _log.info('-> Importing media...') - - media_cache = BasicFileStorage( - args._cache_path['media']) - - # TODO: Add import of queue files - queue_cache = BasicFileStorage(args._cache_path['queue']) - - for entry in db.MediaEntry.query.filter_by(): - for name, path in entry.media_files.items(): - _log.info('Importing: {0} - {1}'.format( - entry.title.encode('ascii', 'replace'), - name)) - - media_file = mg_globals.public_store.get_file(path, mode='wb') - media_file.write( - media_cache.get_file(path, mode='rb').read()) - - _log.info('...Media imported') - - -def _import_database(db, args): - ''' - Restore mongo database from ___.bson files - ''' - _log.info('-> Importing database...') - - p = subprocess.Popen([ - args.mongorestore_path, - '-d', db.name, - os.path.join(args._cache_path['database'], db.name)]) - - p.wait() - - _log.info('...Database imported') - - -def env_import(args): - ''' - Restore mongo database and media files from a tar archive - ''' - if not args.cache_path: - args.cache_path = tempfile.mkdtemp() - - setup_global_and_app_config(args.conf_file) - - # Creates mg_globals.public_store and mg_globals.queue_store - setup_storage() - - global_config, app_config = setup_global_and_app_config(args.conf_file) - db = setup_connection_and_db_from_config( - app_config) - - tf = tarfile.open( - args.tar_file, - mode='r|gz') - - tf.extractall(args.cache_path) - - args.cache_path = os.path.join( - args.cache_path, 'mediagoblin-data') - args = _setup_paths(args) - - # Import database from extracted data - _import_database(db, args) - - _import_media(db, args) - - _clean(args) - - -def _setup_paths(args): - ''' - Populate ``args`` variable with cache subpaths - ''' - args._cache_path = dict() - PATH_MAP = { - 'media': 'media', - 'queue': 'queue', - 'database': 'database'} - - for key, val in PATH_MAP.items(): - args._cache_path[key] = os.path.join(args.cache_path, val) - - return args - - -def _create_archive(args): - ''' - Create the tar archive - ''' - _log.info('-> Compressing to archive') - - tf = tarfile.open( - args.tar_file, - mode='w|gz') - - with closing(tf): - tf.add(args.cache_path, 'mediagoblin-data/') - - _log.info('...Archiving done') - - -def _clean(args): - ''' - Remove cache directory - ''' - shutil.rmtree(args.cache_path) - - -def _export_check(args): - ''' - Run security checks for export command - ''' - if os.path.exists(args.tar_file): - overwrite = raw_input( - 'The output file already exists. ' - 'Are you **SURE** you want to overwrite it? ' - '(yes/no)> ') - if not overwrite == 'yes': - print 'Aborting.' - - return False - - return True - - -def _export_database(db, args): - _log.info('-> Exporting database...') - - p = subprocess.Popen([ - args.mongodump_path, - '-d', db.name, - '-o', args._cache_path['database']]) - - p.wait() - - _log.info('...Database exported') - - -def _export_media(db, args): - _log.info('-> Exporting media...') - - media_cache = BasicFileStorage( - args._cache_path['media']) - - # TODO: Add export of queue files - queue_cache = BasicFileStorage(args._cache_path['queue']) - - for entry in db.MediaEntry.query.filter_by(): - for name, path in entry.media_files.items(): - _log.info(u'Exporting {0} - {1}'.format( - entry.title, - name)) - try: - mc_file = media_cache.get_file(path, mode='wb') - mc_file.write( - mg_globals.public_store.get_file(path, mode='rb').read()) - except Exception as e: - _log.error('Failed: {0}'.format(e)) - - _log.info('...Media exported') - - -def env_export(args): - ''' - Export database and media files to a tar archive - ''' - commands_util.check_unrecognized_args(args) - if args.cache_path: - if os.path.exists(args.cache_path): - _log.error('The cache directory must not exist ' - 'before you run this script') - _log.error('Cache directory: {0}'.format(args.cache_path)) - - return False - else: - args.cache_path = tempfile.mkdtemp() - - args = _setup_paths(args) - - if not _export_check(args): - _log.error('Checks did not pass, exiting') - sys.exit(0) - - globa_config, app_config = setup_global_and_app_config(args.conf_file) - - setup_storage() - - db = setup_connection_and_db_from_config(app_config) - - _export_database(db, args) - - _export_media(db, args) - - _create_archive(args) - - _clean(args)