Commit | Line | Data |
---|---|---|
e86d4f5d | 1 | # GNU MediaGoblin -- federated, autonomous media hosting |
cf29e8a8 | 2 | # Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. |
e86d4f5d JW |
3 | # |
4 | # This program is free software: you can redistribute it and/or modify | |
5 | # it under the terms of the GNU Affero General Public License as published by | |
6 | # the Free Software Foundation, either version 3 of the License, or | |
7 | # (at your option) any later version. | |
8 | # | |
9 | # This program is distributed in the hope that it will be useful, | |
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | # GNU Affero General Public License for more details. | |
13 | # | |
14 | # You should have received a copy of the GNU Affero General Public License | |
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | ||
e86d4f5d | 17 | from mediagoblin import mg_globals |
e86d4f5d | 18 | from mediagoblin.db.open import setup_connection_and_db_from_config |
fd10c716 | 19 | from mediagoblin.storage.filestorage import BasicFileStorage |
2a233ae3 | 20 | from mediagoblin.init import setup_storage, setup_global_and_app_config |
e86d4f5d | 21 | |
7219983f | 22 | import shutil |
e86d4f5d | 23 | import tarfile |
6c6009ba | 24 | import tempfile |
e86d4f5d JW |
25 | import subprocess |
26 | import os.path | |
2a233ae3 | 27 | import os |
7219983f | 28 | import sys |
33d3de8e | 29 | import logging |
c02bea6f | 30 | from contextlib import closing |
7219983f | 31 | |
33d3de8e JW |
32 | _log = logging.getLogger('gmg.import_export') |
33 | logging.basicConfig() | |
34 | _log.setLevel(logging.INFO) | |
35 | ||
e86d4f5d JW |
36 | |
37 | def import_export_parse_setup(subparser): | |
38 | # TODO: Add default | |
39 | subparser.add_argument( | |
40 | 'tar_file') | |
e86d4f5d JW |
41 | subparser.add_argument( |
42 | '--mongodump_path', default='mongodump', | |
43 | help='mongodump binary') | |
224813d2 JW |
44 | subparser.add_argument( |
45 | '--mongorestore_path', default='mongorestore', | |
46 | help='mongorestore binary') | |
47 | subparser.add_argument( | |
6c6009ba CAW |
48 | '--cache_path', |
49 | help='Temporary directory where files will be temporarily dumped') | |
e86d4f5d | 50 | |
7219983f | 51 | |
7219983f | 52 | def _import_media(db, args): |
33d3de8e | 53 | ''' |
7219983f JW |
54 | Import media files |
55 | ||
56 | Must be called after _import_database() | |
33d3de8e JW |
57 | ''' |
58 | _log.info('-> Importing media...') | |
7219983f JW |
59 | |
60 | media_cache = BasicFileStorage( | |
61 | args._cache_path['media']) | |
62 | ||
63 | # TODO: Add import of queue files | |
a855e92a | 64 | queue_cache = BasicFileStorage(args._cache_path['queue']) |
2a233ae3 | 65 | |
8eb21638 | 66 | for entry in db.MediaEntry.find(): |
228c4470 | 67 | for name, path in entry.media_files.items(): |
33d3de8e | 68 | _log.info('Importing: {0} - {1}'.format( |
c8bee965 | 69 | entry.title.encode('ascii', 'replace'), |
33d3de8e JW |
70 | name)) |
71 | ||
7219983f JW |
72 | media_file = mg_globals.public_store.get_file(path, mode='wb') |
73 | media_file.write( | |
74 | media_cache.get_file(path, mode='rb').read()) | |
75 | ||
33d3de8e | 76 | _log.info('...Media imported') |
2a233ae3 | 77 | |
2a233ae3 | 78 | |
e86d4f5d | 79 | def _import_database(db, args): |
33d3de8e | 80 | ''' |
8f12c9b2 | 81 | Restore mongo database from ___.bson files |
33d3de8e JW |
82 | ''' |
83 | _log.info('-> Importing database...') | |
7219983f | 84 | |
8f12c9b2 JW |
85 | p = subprocess.Popen([ |
86 | args.mongorestore_path, | |
87 | '-d', db.name, | |
88 | os.path.join(args._cache_path['database'], db.name)]) | |
243c3843 | 89 | |
7219983f JW |
90 | p.wait() |
91 | ||
33d3de8e | 92 | _log.info('...Database imported') |
e86d4f5d | 93 | |
7219983f | 94 | |
8f12c9b2 | 95 | def env_import(args): |
33d3de8e | 96 | ''' |
8f12c9b2 | 97 | Restore mongo database and media files from a tar archive |
33d3de8e | 98 | ''' |
6c6009ba CAW |
99 | if not args.cache_path: |
100 | args.cache_path = tempfile.mkdtemp() | |
101 | ||
7219983f | 102 | setup_global_and_app_config(args.conf_file) |
8f12c9b2 JW |
103 | |
104 | # Creates mg_globals.public_store and mg_globals.queue_store | |
7219983f JW |
105 | setup_storage() |
106 | ||
33d3de8e | 107 | global_config, app_config = setup_global_and_app_config(args.conf_file) |
bc142abc | 108 | db = setup_connection_and_db_from_config( |
8eb21638 | 109 | app_config) |
e86d4f5d | 110 | |
224813d2 JW |
111 | tf = tarfile.open( |
112 | args.tar_file, | |
113 | mode='r|gz') | |
9188f3af | 114 | |
7219983f JW |
115 | tf.extractall(args.cache_path) |
116 | ||
2db2211d CAW |
117 | args.cache_path = os.path.join( |
118 | args.cache_path, 'mediagoblin-data') | |
8f12c9b2 JW |
119 | args = _setup_paths(args) |
120 | ||
7219983f JW |
121 | # Import database from extracted data |
122 | _import_database(db, args) | |
123 | ||
124 | _import_media(db, args) | |
125 | ||
00e381f7 CAW |
126 | _clean(args) |
127 | ||
224813d2 | 128 | |
2a233ae3 | 129 | def _setup_paths(args): |
33d3de8e | 130 | ''' |
8f12c9b2 | 131 | Populate ``args`` variable with cache subpaths |
33d3de8e | 132 | ''' |
2a233ae3 JW |
133 | args._cache_path = dict() |
134 | PATH_MAP = { | |
135 | 'media': 'media', | |
7219983f | 136 | 'queue': 'queue', |
2a233ae3 | 137 | 'database': 'database'} |
7219983f | 138 | |
2a233ae3 JW |
139 | for key, val in PATH_MAP.items(): |
140 | args._cache_path[key] = os.path.join(args.cache_path, val) | |
141 | ||
142 | return args | |
143 | ||
7219983f | 144 | |
2a233ae3 | 145 | def _create_archive(args): |
33d3de8e | 146 | ''' |
8f12c9b2 | 147 | Create the tar archive |
33d3de8e JW |
148 | ''' |
149 | _log.info('-> Compressing to archive') | |
7219983f | 150 | |
2a233ae3 JW |
151 | tf = tarfile.open( |
152 | args.tar_file, | |
153 | mode='w|gz') | |
2a233ae3 | 154 | |
c02bea6f | 155 | with closing(tf): |
7219983f JW |
156 | tf.add(args.cache_path, 'mediagoblin-data/') |
157 | ||
33d3de8e | 158 | _log.info('...Archiving done') |
7219983f JW |
159 | |
160 | ||
161 | def _clean(args): | |
33d3de8e | 162 | ''' |
8f12c9b2 | 163 | Remove cache directory |
33d3de8e | 164 | ''' |
7219983f | 165 | shutil.rmtree(args.cache_path) |
2a233ae3 | 166 | |
2a233ae3 | 167 | |
8f12c9b2 | 168 | def _export_check(args): |
33d3de8e | 169 | ''' |
8f12c9b2 | 170 | Run security checks for export command |
33d3de8e | 171 | ''' |
7219983f JW |
172 | if os.path.exists(args.tar_file): |
173 | overwrite = raw_input( | |
174 | 'The output file already exists. ' | |
175 | 'Are you **SURE** you want to overwrite it? ' | |
176 | '(yes/no)> ') | |
177 | if not overwrite == 'yes': | |
33d3de8e | 178 | print 'Aborting.' |
2a233ae3 | 179 | |
7219983f | 180 | return False |
2a233ae3 | 181 | |
7219983f | 182 | return True |
2a233ae3 | 183 | |
2a233ae3 | 184 | |
8f12c9b2 | 185 | def _export_database(db, args): |
33d3de8e | 186 | _log.info('-> Exporting database...') |
8f12c9b2 JW |
187 | |
188 | p = subprocess.Popen([ | |
189 | args.mongodump_path, | |
190 | '-d', db.name, | |
191 | '-o', args._cache_path['database']]) | |
192 | ||
193 | p.wait() | |
194 | ||
33d3de8e | 195 | _log.info('...Database exported') |
8f12c9b2 JW |
196 | |
197 | ||
198 | def _export_media(db, args): | |
33d3de8e | 199 | _log.info('-> Exporting media...') |
8f12c9b2 JW |
200 | |
201 | media_cache = BasicFileStorage( | |
202 | args._cache_path['media']) | |
203 | ||
204 | # TODO: Add export of queue files | |
a855e92a | 205 | queue_cache = BasicFileStorage(args._cache_path['queue']) |
8f12c9b2 | 206 | |
8eb21638 | 207 | for entry in db.MediaEntry.find(): |
228c4470 | 208 | for name, path in entry.media_files.items(): |
a9c7af90 | 209 | _log.info(u'Exporting {0} - {1}'.format( |
ec82fbd8 | 210 | entry.title, |
33d3de8e | 211 | name)) |
a7ca2a72 JW |
212 | try: |
213 | mc_file = media_cache.get_file(path, mode='wb') | |
214 | mc_file.write( | |
215 | mg_globals.public_store.get_file(path, mode='rb').read()) | |
8eb21638 | 216 | except Exception as e: |
a7ca2a72 | 217 | _log.error('Failed: {0}'.format(e)) |
8f12c9b2 | 218 | |
33d3de8e | 219 | _log.info('...Media exported') |
8f12c9b2 JW |
220 | |
221 | ||
e86d4f5d | 222 | def env_export(args): |
33d3de8e | 223 | ''' |
8f12c9b2 | 224 | Export database and media files to a tar archive |
33d3de8e | 225 | ''' |
6c6009ba CAW |
226 | if args.cache_path: |
227 | if os.path.exists(args.cache_path): | |
243c3843 NY |
228 | _log.error('The cache directory must not exist ' |
229 | 'before you run this script') | |
33d3de8e | 230 | _log.error('Cache directory: {0}'.format(args.cache_path)) |
6c6009ba CAW |
231 | |
232 | return False | |
233 | else: | |
234 | args.cache_path = tempfile.mkdtemp() | |
235 | ||
2a233ae3 JW |
236 | args = _setup_paths(args) |
237 | ||
8f12c9b2 | 238 | if not _export_check(args): |
33d3de8e | 239 | _log.error('Checks did not pass, exiting') |
7219983f JW |
240 | sys.exit(0) |
241 | ||
33d3de8e | 242 | globa_config, app_config = setup_global_and_app_config(args.conf_file) |
2a233ae3 | 243 | |
33d3de8e | 244 | setup_storage() |
243c3843 | 245 | |
bc142abc | 246 | db = setup_connection_and_db_from_config(app_config) |
e86d4f5d | 247 | |
2a233ae3 | 248 | _export_database(db, args) |
e86d4f5d | 249 | |
2a233ae3 | 250 | _export_media(db, args) |
e86d4f5d | 251 | |
2a233ae3 | 252 | _create_archive(args) |
7219983f JW |
253 | |
254 | _clean(args) |