Commit | Line | Data |
---|---|---|
e86d4f5d | 1 | # GNU MediaGoblin -- federated, autonomous media hosting |
cf29e8a8 | 2 | # Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. |
e86d4f5d JW |
3 | # |
4 | # This program is free software: you can redistribute it and/or modify | |
5 | # it under the terms of the GNU Affero General Public License as published by | |
6 | # the Free Software Foundation, either version 3 of the License, or | |
7 | # (at your option) any later version. | |
8 | # | |
9 | # This program is distributed in the hope that it will be useful, | |
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | # GNU Affero General Public License for more details. | |
13 | # | |
14 | # You should have received a copy of the GNU Affero General Public License | |
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | ||
e86d4f5d | 17 | from mediagoblin import mg_globals |
e86d4f5d | 18 | from mediagoblin.db.open import setup_connection_and_db_from_config |
fd10c716 | 19 | from mediagoblin.storage.filestorage import BasicFileStorage |
2a233ae3 | 20 | from mediagoblin.init import setup_storage, setup_global_and_app_config |
e86d4f5d | 21 | |
7219983f | 22 | import shutil |
e86d4f5d | 23 | import tarfile |
6c6009ba | 24 | import tempfile |
e86d4f5d JW |
25 | import subprocess |
26 | import os.path | |
2a233ae3 | 27 | import os |
7219983f | 28 | import sys |
33d3de8e | 29 | import logging |
c02bea6f | 30 | from contextlib import closing |
7219983f | 31 | |
33d3de8e JW |
32 | _log = logging.getLogger('gmg.import_export') |
33 | logging.basicConfig() | |
34 | _log.setLevel(logging.INFO) | |
35 | ||
e86d4f5d JW |
36 | |
37 | def import_export_parse_setup(subparser): | |
38 | # TODO: Add default | |
39 | subparser.add_argument( | |
40 | 'tar_file') | |
e86d4f5d JW |
41 | subparser.add_argument( |
42 | '--mongodump_path', default='mongodump', | |
43 | help='mongodump binary') | |
224813d2 JW |
44 | subparser.add_argument( |
45 | '--mongorestore_path', default='mongorestore', | |
46 | help='mongorestore binary') | |
47 | subparser.add_argument( | |
6c6009ba CAW |
48 | '--cache_path', |
49 | help='Temporary directory where files will be temporarily dumped') | |
e86d4f5d | 50 | |
7219983f | 51 | |
7219983f | 52 | def _import_media(db, args): |
33d3de8e | 53 | ''' |
7219983f JW |
54 | Import media files |
55 | ||
56 | Must be called after _import_database() | |
33d3de8e JW |
57 | ''' |
58 | _log.info('-> Importing media...') | |
7219983f JW |
59 | |
60 | media_cache = BasicFileStorage( | |
61 | args._cache_path['media']) | |
62 | ||
63 | # TODO: Add import of queue files | |
2a233ae3 JW |
64 | queue_cache = BasicFileStorage( |
65 | args._cache_path['queue']) | |
66 | ||
8eb21638 | 67 | for entry in db.MediaEntry.find(): |
228c4470 | 68 | for name, path in entry.media_files.items(): |
33d3de8e | 69 | _log.info('Importing: {0} - {1}'.format( |
ec82fbd8 | 70 | entry.title, |
33d3de8e JW |
71 | name)) |
72 | ||
7219983f JW |
73 | media_file = mg_globals.public_store.get_file(path, mode='wb') |
74 | media_file.write( | |
75 | media_cache.get_file(path, mode='rb').read()) | |
76 | ||
33d3de8e | 77 | _log.info('...Media imported') |
2a233ae3 | 78 | |
2a233ae3 | 79 | |
e86d4f5d | 80 | def _import_database(db, args): |
33d3de8e | 81 | ''' |
8f12c9b2 | 82 | Restore mongo database from ___.bson files |
33d3de8e JW |
83 | ''' |
84 | _log.info('-> Importing database...') | |
7219983f | 85 | |
8f12c9b2 JW |
86 | p = subprocess.Popen([ |
87 | args.mongorestore_path, | |
88 | '-d', db.name, | |
89 | os.path.join(args._cache_path['database'], db.name)]) | |
243c3843 | 90 | |
7219983f JW |
91 | p.wait() |
92 | ||
33d3de8e | 93 | _log.info('...Database imported') |
e86d4f5d | 94 | |
7219983f | 95 | |
8f12c9b2 | 96 | def env_import(args): |
33d3de8e | 97 | ''' |
8f12c9b2 | 98 | Restore mongo database and media files from a tar archive |
33d3de8e | 99 | ''' |
6c6009ba CAW |
100 | if not args.cache_path: |
101 | args.cache_path = tempfile.mkdtemp() | |
102 | ||
7219983f | 103 | setup_global_and_app_config(args.conf_file) |
8f12c9b2 JW |
104 | |
105 | # Creates mg_globals.public_store and mg_globals.queue_store | |
7219983f JW |
106 | setup_storage() |
107 | ||
33d3de8e | 108 | global_config, app_config = setup_global_and_app_config(args.conf_file) |
e86d4f5d | 109 | connection, db = setup_connection_and_db_from_config( |
8eb21638 | 110 | app_config) |
e86d4f5d | 111 | |
224813d2 JW |
112 | tf = tarfile.open( |
113 | args.tar_file, | |
114 | mode='r|gz') | |
9188f3af | 115 | |
7219983f JW |
116 | tf.extractall(args.cache_path) |
117 | ||
2db2211d CAW |
118 | args.cache_path = os.path.join( |
119 | args.cache_path, 'mediagoblin-data') | |
8f12c9b2 JW |
120 | args = _setup_paths(args) |
121 | ||
7219983f JW |
122 | # Import database from extracted data |
123 | _import_database(db, args) | |
124 | ||
125 | _import_media(db, args) | |
126 | ||
00e381f7 CAW |
127 | _clean(args) |
128 | ||
224813d2 | 129 | |
2a233ae3 | 130 | def _setup_paths(args): |
33d3de8e | 131 | ''' |
8f12c9b2 | 132 | Populate ``args`` variable with cache subpaths |
33d3de8e | 133 | ''' |
2a233ae3 JW |
134 | args._cache_path = dict() |
135 | PATH_MAP = { | |
136 | 'media': 'media', | |
7219983f | 137 | 'queue': 'queue', |
2a233ae3 | 138 | 'database': 'database'} |
7219983f | 139 | |
2a233ae3 JW |
140 | for key, val in PATH_MAP.items(): |
141 | args._cache_path[key] = os.path.join(args.cache_path, val) | |
142 | ||
143 | return args | |
144 | ||
7219983f | 145 | |
2a233ae3 | 146 | def _create_archive(args): |
33d3de8e | 147 | ''' |
8f12c9b2 | 148 | Create the tar archive |
33d3de8e JW |
149 | ''' |
150 | _log.info('-> Compressing to archive') | |
7219983f | 151 | |
2a233ae3 JW |
152 | tf = tarfile.open( |
153 | args.tar_file, | |
154 | mode='w|gz') | |
2a233ae3 | 155 | |
c02bea6f | 156 | with closing(tf): |
7219983f JW |
157 | tf.add(args.cache_path, 'mediagoblin-data/') |
158 | ||
33d3de8e | 159 | _log.info('...Archiving done') |
7219983f JW |
160 | |
161 | ||
162 | def _clean(args): | |
33d3de8e | 163 | ''' |
8f12c9b2 | 164 | Remove cache directory |
33d3de8e | 165 | ''' |
7219983f | 166 | shutil.rmtree(args.cache_path) |
2a233ae3 | 167 | |
2a233ae3 | 168 | |
8f12c9b2 | 169 | def _export_check(args): |
33d3de8e | 170 | ''' |
8f12c9b2 | 171 | Run security checks for export command |
33d3de8e | 172 | ''' |
7219983f JW |
173 | if os.path.exists(args.tar_file): |
174 | overwrite = raw_input( | |
175 | 'The output file already exists. ' | |
176 | 'Are you **SURE** you want to overwrite it? ' | |
177 | '(yes/no)> ') | |
178 | if not overwrite == 'yes': | |
33d3de8e | 179 | print 'Aborting.' |
2a233ae3 | 180 | |
7219983f | 181 | return False |
2a233ae3 | 182 | |
7219983f | 183 | return True |
2a233ae3 | 184 | |
2a233ae3 | 185 | |
8f12c9b2 | 186 | def _export_database(db, args): |
33d3de8e | 187 | _log.info('-> Exporting database...') |
8f12c9b2 JW |
188 | |
189 | p = subprocess.Popen([ | |
190 | args.mongodump_path, | |
191 | '-d', db.name, | |
192 | '-o', args._cache_path['database']]) | |
193 | ||
194 | p.wait() | |
195 | ||
33d3de8e | 196 | _log.info('...Database exported') |
8f12c9b2 JW |
197 | |
198 | ||
199 | def _export_media(db, args): | |
33d3de8e | 200 | _log.info('-> Exporting media...') |
8f12c9b2 JW |
201 | |
202 | media_cache = BasicFileStorage( | |
203 | args._cache_path['media']) | |
204 | ||
205 | # TODO: Add export of queue files | |
206 | queue_cache = BasicFileStorage( | |
207 | args._cache_path['queue']) | |
208 | ||
8eb21638 | 209 | for entry in db.MediaEntry.find(): |
228c4470 | 210 | for name, path in entry.media_files.items(): |
a9c7af90 | 211 | _log.info(u'Exporting {0} - {1}'.format( |
ec82fbd8 | 212 | entry.title, |
33d3de8e | 213 | name)) |
a7ca2a72 JW |
214 | try: |
215 | mc_file = media_cache.get_file(path, mode='wb') | |
216 | mc_file.write( | |
217 | mg_globals.public_store.get_file(path, mode='rb').read()) | |
8eb21638 | 218 | except Exception as e: |
a7ca2a72 | 219 | _log.error('Failed: {0}'.format(e)) |
8f12c9b2 | 220 | |
33d3de8e | 221 | _log.info('...Media exported') |
8f12c9b2 JW |
222 | |
223 | ||
e86d4f5d | 224 | def env_export(args): |
33d3de8e | 225 | ''' |
8f12c9b2 | 226 | Export database and media files to a tar archive |
33d3de8e | 227 | ''' |
6c6009ba CAW |
228 | if args.cache_path: |
229 | if os.path.exists(args.cache_path): | |
243c3843 NY |
230 | _log.error('The cache directory must not exist ' |
231 | 'before you run this script') | |
33d3de8e | 232 | _log.error('Cache directory: {0}'.format(args.cache_path)) |
6c6009ba CAW |
233 | |
234 | return False | |
235 | else: | |
236 | args.cache_path = tempfile.mkdtemp() | |
237 | ||
2a233ae3 JW |
238 | args = _setup_paths(args) |
239 | ||
8f12c9b2 | 240 | if not _export_check(args): |
33d3de8e | 241 | _log.error('Checks did not pass, exiting') |
7219983f JW |
242 | sys.exit(0) |
243 | ||
33d3de8e | 244 | globa_config, app_config = setup_global_and_app_config(args.conf_file) |
2a233ae3 | 245 | |
33d3de8e | 246 | setup_storage() |
243c3843 | 247 | |
e86d4f5d | 248 | connection, db = setup_connection_and_db_from_config( |
8eb21638 | 249 | app_config) |
e86d4f5d | 250 | |
2a233ae3 | 251 | _export_database(db, args) |
e86d4f5d | 252 | |
2a233ae3 | 253 | _export_media(db, args) |
e86d4f5d | 254 | |
2a233ae3 | 255 | _create_archive(args) |
7219983f JW |
256 | |
257 | _clean(args) |