1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects.
26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library.
31 """
32
33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon
35 from translate.misc import quote
36 from translate.lang import data
37 from ctypes import *
38 import ctypes.util
39 try:
40 import cStringIO as StringIO
41 except ImportError:
42 import StringIO
43 import os
44 import pypo
45 import re
46 import sys
47 import tempfile
48
49 lsep = " "
50 """Seperator for #: entries"""
51
52 STRING = c_char_p
53
54
57
58
59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING)
61
62
63
65 _fields_ = [('xerror', xerror_prototype),
66 ('xerror2', xerror2_prototype)]
67
69 _fields_ = [
70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
74 ]
75
76
77 -def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_text):
78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, column, multilint_p, message_text
79 if severity >= 1:
80 raise ValueError(message_text)
81
82 -def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2):
83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, column1, multiline_p1, message_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2
84 if severity >= 1:
85 raise ValueError(message_text1)
86
87
88
89
90 gpo = None
91
92
93 names = ['gettextpo', 'libgettextpo']
94 for name in names:
95 lib_location = ctypes.util.find_library(name)
96 if lib_location:
97 gpo = cdll.LoadLibrary(lib_location)
98 if gpo:
99 break
100 else:
101
102
103 try:
104 gpo = cdll.LoadLibrary('libgettextpo.so')
105 except OSError, e:
106 raise ImportError("gettext PO library not found")
107
108
109
110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
112 gpo.po_file_write_v2.retype = c_int
113
114
115 gpo.po_file_domain_header.restype = STRING
116 gpo.po_header_field.restype = STRING
117 gpo.po_header_field.argtypes = [STRING, STRING]
118
119
120 gpo.po_filepos_file.restype = STRING
121 gpo.po_message_filepos.restype = c_int
122 gpo.po_message_filepos.argtypes = [c_int, c_int]
123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int]
124
125
126 gpo.po_message_comments.restype = STRING
127 gpo.po_message_extracted_comments.restype = STRING
128 gpo.po_message_prev_msgctxt.restype = STRING
129 gpo.po_message_prev_msgid.restype = STRING
130 gpo.po_message_prev_msgid_plural.restype = STRING
131 gpo.po_message_is_format.restype = c_int
132 gpo.po_message_msgctxt.restype = STRING
133 gpo.po_message_msgid.restype = STRING
134 gpo.po_message_msgid_plural.restype = STRING
135 gpo.po_message_msgstr.restype = STRING
136 gpo.po_message_msgstr_plural.restype = STRING
137
138
139 gpo.po_message_set_comments.argtypes = [c_int, STRING]
140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
143
144
145 xerror_handler = po_xerror_handler()
146 xerror_handler.xerror = xerror_prototype(xerror_cb)
147 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
148
151
154
157
160
162 """Returns the libgettextpo version
163
164 @return: a three-value tuple containing the libgettextpo version in the
165 following format:
166 (major version, minor version, subminor version)
167 """
168 libversion = c_long.in_dll(gpo, 'libgettextpo_version')
169 major = libversion.value >> 16
170 minor = libversion.value >> 8
171 subminor = libversion.value - (major << 16) - (minor << 8)
172 return major, minor, subminor
173
174
175 -class pounit(pocommon.pounit):
196 msgid_plural = property(None, setmsgid_plural)
197
199 def remove_msgid_comments(text):
200 if not text:
201 return text
202 if text.startswith("_:"):
203 remainder = re.search(r"_: .*\n(.*)", text)
204 if remainder:
205 return remainder.group(1)
206 else:
207 return u""
208 else:
209 return text
210 singular = remove_msgid_comments(gpo.po_message_msgid(self._gpo_message))
211 if singular:
212 multi = multistring(singular, self._encoding)
213 if self.hasplural():
214 pluralform = gpo.po_message_msgid_plural(self._gpo_message)
215 if isinstance(pluralform, str):
216 pluralform = pluralform.decode(self._encoding)
217 multi.strings.append(pluralform)
218 return multi
219 else:
220 return u""
221
234
235 source = property(getsource, setsource)
236
238 if self.hasplural():
239 plurals = []
240 nplural = 0
241 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
242 while plural:
243 plurals.append(plural)
244 nplural += 1
245 plural = gpo.po_message_msgstr_plural(self._gpo_message, nplural)
246 if plurals:
247 multi = multistring(plurals, encoding=self._encoding)
248 else:
249 multi = multistring(u"")
250 else:
251 multi = multistring(gpo.po_message_msgstr(self._gpo_message) or u"", encoding=self._encoding)
252 return multi
253
255
256 if self.hasplural():
257 if isinstance(target, multistring):
258 target = target.strings
259 elif isinstance(target, basestring):
260 target = [target]
261
262 elif isinstance(target,(dict, list)):
263 if len(target) == 1:
264 target = target[0]
265 else:
266 raise ValueError("po msgid element has no plural but msgstr has %d elements (%s)" % (len(target), target))
267
268
269
270
271
272 if isinstance(target, (dict, list)):
273 i = 0
274 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
275 while message is not None:
276 gpo.po_message_set_msgstr_plural(self._gpo_message, i, None)
277 i += 1
278 message = gpo.po_message_msgstr_plural(self._gpo_message, i)
279
280 if isinstance(target, list):
281 for i in range(len(target)):
282 targetstring = target[i]
283 if isinstance(targetstring, unicode):
284 targetstring = targetstring.encode(self._encoding)
285 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
286
287 elif isinstance(target, dict):
288 for i, targetstring in enumerate(target.itervalues()):
289 gpo.po_message_set_msgstr_plural(self._gpo_message, i, targetstring)
290
291 else:
292 if isinstance(target, unicode):
293 target = target.encode(self._encoding)
294 if target is None:
295 gpo.po_message_set_msgstr(self._gpo_message, "")
296 else:
297 gpo.po_message_set_msgstr(self._gpo_message, target)
298 target = property(gettarget, settarget)
299
301 """The unique identifier for this unit according to the convensions in
302 .mo files."""
303 id = gpo.po_message_msgid(self._gpo_message)
304
305
306
307
308
309
310
311 context = gpo.po_message_msgctxt(self._gpo_message)
312 if context:
313 id = "%s\04%s" % (context, id)
314 return id or ""
315
317 if origin == None:
318 comments = gpo.po_message_comments(self._gpo_message) + \
319 gpo.po_message_extracted_comments(self._gpo_message)
320 elif origin == "translator":
321 comments = gpo.po_message_comments(self._gpo_message)
322 elif origin in ["programmer", "developer", "source code"]:
323 comments = gpo.po_message_extracted_comments(self._gpo_message)
324 else:
325 raise ValueError("Comment type not valid")
326
327 if comments and get_libgettextpo_version() < (0, 17, 0):
328 comments = "\n".join([line.strip() for line in comments.split("\n")])
329
330 return comments[:-1].decode(self._encoding)
331
332 - def addnote(self, text, origin=None, position="append"):
333
334 if (not text) or (not text.strip()):
335 return
336 text = data.forceunicode(text)
337 oldnotes = self.getnotes(origin)
338 newnotes = None
339 if oldnotes:
340 if position == "append":
341 newnotes = oldnotes + "\n" + text
342 elif position == "merge":
343 if oldnotes != text:
344 oldnoteslist = oldnotes.split("\n")
345 for newline in text.split("\n"):
346 newline = newline.rstrip()
347
348 if newline not in oldnotes or len(newline) < 5:
349 oldnoteslist.append(newline)
350 newnotes = "\n".join(oldnoteslist)
351 else:
352 newnotes = text + '\n' + oldnotes
353 else:
354 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
355
356 if newnotes:
357 newlines = []
358 needs_space = get_libgettextpo_version() < (0, 17, 0)
359 for line in newnotes.split("\n"):
360 if line and needs_space:
361 newlines.append(" " + line)
362 else:
363 newlines.append(line)
364 newnotes = "\n".join(newlines)
365 if origin in ["programmer", "developer", "source code"]:
366 gpo.po_message_set_extracted_comments(self._gpo_message, newnotes)
367 else:
368 gpo.po_message_set_comments(self._gpo_message, newnotes)
369
371 gpo.po_message_set_comments(self._gpo_message, "")
372
374 newpo = self.__class__()
375 newpo._gpo_message = self._gpo_message
376 return newpo
377
378 - def merge(self, otherpo, overwrite=False, comments=True, authoritative=False):
413
415
416
417 return self.getid() == "" and len(self.target) > 0
418
421
424
431
434
437
439 return gpo.po_message_is_fuzzy(self._gpo_message)
440
442 gpo.po_message_set_fuzzy(self._gpo_message, present)
443
446
448 return gpo.po_message_is_obsolete(self._gpo_message)
449
451
452
453 gpo.po_message_set_obsolete(self._gpo_message, True)
454
456 gpo.po_message_set_obsolete(self._gpo_message, False)
457
459 return gpo.po_message_msgid_plural(self._gpo_message) is not None
460
476
481
483 locations = []
484 i = 0
485 location = gpo.po_message_filepos(self._gpo_message, i)
486 while location:
487 locname = gpo.po_filepos_file(location)
488 locline = gpo.po_filepos_start_line(location)
489 if locline == -1:
490 locstring = locname
491 else:
492 locstring = locname + ":" + str(locline)
493 locations.append(locstring)
494 i += 1
495 location = gpo.po_message_filepos(self._gpo_message, i)
496 return locations
497
499 for loc in location.split():
500 parts = loc.split(":")
501 file = parts[0]
502 if len(parts) == 2:
503 line = int(parts[1])
504 else:
505 line = -1
506 gpo.po_message_add_filepos(self._gpo_message, file, line)
507
508 - def getcontext(self):
509 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
510 msgidcomment = self._extract_msgidcomments()
511 if msgctxt:
512 return msgctxt + msgidcomment
513 else:
514 return msgidcomment
515
516 -class pofile(pocommon.pofile):
517 UnitClass = pounit
519 self.UnitClass = unitclass
520 pocommon.pofile.__init__(self, unitclass=unitclass)
521 self._gpo_memory_file = None
522 self._gpo_message_iterator = None
523 self._encoding = encodingToUse(encoding)
524 if inputfile is not None:
525 self.parse(inputfile)
526 else:
527 self._gpo_memory_file = gpo.po_file_create()
528 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
529
531 gpo.po_message_insert(self._gpo_message_iterator, unit._gpo_message)
532 self.units.append(unit)
533
535 """make sure each msgid is unique ; merge comments etc from duplicates into original"""
536 msgiddict = {}
537 uniqueunits = []
538
539
540 markedpos = []
541 def addcomment(thepo):
542 thepo.msgidcomment = " ".join(thepo.getlocations())
543 markedpos.append(thepo)
544 for thepo in self.units:
545 if thepo.isheader():
546 uniqueunits.append(thepo)
547 continue
548 if duplicatestyle.startswith("msgid_comment"):
549 msgid = thepo._extract_msgidcomments() + thepo.source
550 else:
551 msgid = thepo.source
552 if duplicatestyle == "msgid_comment_all":
553 addcomment(thepo)
554 uniqueunits.append(thepo)
555 elif msgid in msgiddict:
556 if duplicatestyle == "merge":
557 if msgid:
558 msgiddict[msgid].merge(thepo)
559 else:
560 addcomment(thepo)
561 uniqueunits.append(thepo)
562 elif duplicatestyle == "keep":
563 uniqueunits.append(thepo)
564 elif duplicatestyle == "msgid_comment":
565 origpo = msgiddict[msgid]
566 if origpo not in markedpos:
567 addcomment(origpo)
568 addcomment(thepo)
569 uniqueunits.append(thepo)
570 elif duplicatestyle == "msgctxt":
571 origpo = msgiddict[msgid]
572 if origpo not in markedpos:
573 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join(origpo.getlocations()))
574 markedpos.append(thepo)
575 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thepo.getlocations()))
576 uniqueunits.append(thepo)
577 else:
578 if not msgid and duplicatestyle != "keep":
579 addcomment(thepo)
580 msgiddict[msgid] = thepo
581 uniqueunits.append(thepo)
582 new_gpo_memory_file = gpo.po_file_create()
583 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
584 for unit in uniqueunits:
585 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
586 gpo.po_message_iterator_free(self._gpo_message_iterator)
587 self._gpo_message_iterator = new_gpo_message_iterator
588 self._gpo_memory_file = new_gpo_memory_file
589 self.units = uniqueunits
590
592 def obsolete_workaround():
593
594
595
596 for unit in self.units:
597 if unit.isobsolete():
598 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
599 location = gpo.po_message_filepos(unit._gpo_message, 0)
600 while location:
601 gpo.po_message_remove_filepos(unit._gpo_message, 0)
602 location = gpo.po_message_filepos(unit._gpo_message, 0)
603 outputstring = ""
604 if self._gpo_memory_file:
605 obsolete_workaround()
606 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po')
607 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler)
608 f.seek(0)
609 outputstring = f.read()
610 f.close()
611 return outputstring
612
614 """Returns True if the object doesn't contain any translation units."""
615 if len(self.units) == 0:
616 return True
617
618 if self.units[0].isheader():
619 units = self.units[1:]
620 else:
621 units = self.units
622
623 for unit in units:
624 if not unit.isblank() and not unit.isobsolete():
625 return False
626 return True
627
629 if hasattr(input, 'name'):
630 self.filename = input.name
631 elif not getattr(self, 'filename', ''):
632 self.filename = ''
633
634 if hasattr(input, "read"):
635 posrc = input.read()
636 input.close()
637 input = posrc
638
639 needtmpfile = not os.path.isfile(input)
640 if needtmpfile:
641
642 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
643 os.write(fd, input)
644 input = fname
645 os.close(fd)
646
647 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
648 if self._gpo_memory_file is None:
649 print >> sys.stderr, "Error:"
650
651 if needtmpfile:
652 os.remove(input)
653
654
655 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
656 if self._header:
657 charset = gpo.po_header_field(self._header, "Content-Type")
658 if charset:
659 charset = re.search("charset=([^\\s]+)", charset).group(1)
660 self._encoding = encodingToUse(charset)
661 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_file, None)
662 newmessage = gpo.po_next_message(self._gpo_message_iterator)
663 while newmessage:
664 newunit = pounit(gpo_message=newmessage)
665 self.units.append(newunit)
666 newmessage = gpo.po_next_message(self._gpo_message_iterator)
667 self._free_iterator()
668
670
671
672 return
673 self._free_iterator()
674 if self._gpo_memory_file is not None:
675 gpo.po_file_free(self._gpo_memory_file)
676 self._gpo_memory_file = None
677
679
680
681 return
682 if self._gpo_message_iterator is not None:
683 gpo.po_message_iterator_free(self._gpo_message_iterator)
684 self._gpo_message_iterator = None
685