From 24144bb5506f229ff7d9f4264d30c7b51d9fa099 Mon Sep 17 00:00:00 2001 From: Mads Michelsen Date: Mon, 7 Mar 2022 23:52:01 +0100 Subject: [PATCH] initial --- .gitignore | 82 ++++ LICENSE | 675 +++++++++++++++++++++++++++ README.md | 59 +++ setup.py | 47 ++ src/derailleur/__init__.py | 20 + src/derailleur/about.py | 16 + src/derailleur/args/__init__.py | 33 ++ src/derailleur/client/__init__.py | 54 +++ src/derailleur/config/__init__.py | 75 +++ src/derailleur/config/configspec.ini | 56 +++ src/derailleur/history/__init__.py | 44 ++ src/derailleur/loggers/__init__.py | 67 +++ src/derailleur/manager/__init__.py | 161 +++++++ src/derailleur/postproc/__init__.py | 16 + src/derailleur/postproc/audio.py | 218 +++++++++ src/derailleur/postproc/film.py | 61 +++ src/derailleur/postproc/functions.py | 58 +++ src/derailleur/postproc/generic.py | 59 +++ src/derailleur/postproc/keywords.py | 29 ++ src/derailleur/postproc/stats.py | 115 +++++ src/derailleur/postproc/tv.py | 74 +++ src/scripts/derailleur-add | 52 +++ src/scripts/derailleur-feed | 81 ++++ src/scripts/derailleur-manager | 47 ++ src/scripts/derailleur-postproc | 124 +++++ 25 files changed, 2323 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100755 setup.py create mode 100644 src/derailleur/__init__.py create mode 100644 src/derailleur/about.py create mode 100644 src/derailleur/args/__init__.py create mode 100644 src/derailleur/client/__init__.py create mode 100644 src/derailleur/config/__init__.py create mode 100644 src/derailleur/config/configspec.ini create mode 100644 src/derailleur/history/__init__.py create mode 100644 src/derailleur/loggers/__init__.py create mode 100644 src/derailleur/manager/__init__.py create mode 100644 src/derailleur/postproc/__init__.py create mode 100644 src/derailleur/postproc/audio.py create mode 100644 src/derailleur/postproc/film.py create mode 100644 src/derailleur/postproc/functions.py create mode 100644 src/derailleur/postproc/generic.py create mode 100644 src/derailleur/postproc/keywords.py create mode 100644 src/derailleur/postproc/stats.py create mode 100644 src/derailleur/postproc/tv.py create mode 100755 src/scripts/derailleur-add create mode 100755 src/scripts/derailleur-feed create mode 100755 src/scripts/derailleur-manager create mode 100755 src/scripts/derailleur-postproc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..473e37a --- /dev/null +++ b/.gitignore @@ -0,0 +1,82 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +#Ipython Notebook +.ipynb_checkpoints + +# VIM # +# ### # + +# swap +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +# session +Session.vim +# temporary +.netrwhist +*~ +# auto-generated tag files +tags + +# Building +MANIFEST + +# Porting leftovers +*.bak diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..733c072 --- /dev/null +++ b/LICENSE @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..c782ad1 --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +# derailleur +A set of Python 3 scripts and libraries for interacting with Transmission, +derailleur automates all the little tasks involved in keeping Transmission +humming. + +Derailleur consists of four tools: + * **derailleur-add**: manually add magnet links/torrent files + * **derailleur-feed**: add links by way of an rss feed + * **derailleur-manager**: manage seeds and remove (and delete) old downloads + * **derailleur-postproc**: filter and process downloads after completion + +All use a single library (derailleur) to interact with the Transmission daemon. + +## Features + * All-in-one solution: Gets feeds, removes old torrents, cleans up and + organizes the content once it's downloaded. + * Can replace several more complex solutions like flexget and media management + tools + * Works well with Firefox (add), cron (feed, manager) and Transmission's + `script-torrent-done` option (postproc). + * All elements can maintain an individual log for easy oversight of actions + * Simple and comprehensible ini-like configuration + * Postproc recognises music, tv and film and can work with custom, + user-defined download types + * Organizes content in a manner compatible with with media management tools + and HTPC software so that it is automatically recognized. + * Only keeps the files worth having. No more random .urls and .infos littering + up your media folders. + +For more details, please see the [the +wiki](https://git.brokkr.net/brokkr/derailleur/wiki). + +## Installation +From the root derailleur directory (the one with this README file) derailleur +can be installed with pip. If pip is not on your system, please check your +package manager (on debian/ubuntu the package is named 'python3-pip'). + + python3 ./setup.py sdist + sudo pip3 install ./dist/derailleur-[version].tar.gz + +Using pip to install rather than 'setup.py install' allows you to uninstall with: + + sudo pip3 uninstall derailleur + +## Dependencies + * transmissionrpc : For talking to the Transmission client + * feedparser: For use in derailleur-feed to well, parse feeds. + * mutagen : For reading and writing id3/FLAC tags + * unidecode : For transliterating unicode characters into ascii + * configobj : For parsing and validating the ini-like settings file + * xbmc-json (optional) : For alerting local kodi/XBMC instance to the presence + of new media + * requests (optional) : A dependency of xbmc-json (and used in catching kodi + update errors) + +The dependencies can all be installed with: + + sudo pip3 install [package name] + diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..053f67a --- /dev/null +++ b/setup.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2010-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Setup script for derailleur""" + +from distutils.core import setup + + +setup( + name='derailleur', + version='0.31', + license='GPL3', + description=''''Derailleur is a unified collection of tools for the ''' + ''''Transmission bittorrent client''', + long_description='''Derailleur is a unified collection of tools for ''' + '''the Transmission bittorrent client''', + author='Mads Michelsen', + author_email='mail@brokkr.net', + url='https://github.com/brokkr/derailleur', + scripts=['src/scripts/derailleur-add', 'src/scripts/derailleur-manager', + 'src/scripts/derailleur-feed', 'src/scripts/derailleur-postproc'], + packages=['derailleur', 'derailleur.client', 'derailleur.postproc', + 'derailleur.args', 'derailleur.config', 'derailleur.loggers', + 'derailleur.manager', 'derailleur.history'], + package_dir={'' : 'src/'}, + package_data={'derailleur': ['config/configspec.ini']}, + include_package_data=True, + requires=['transmissionrpc', 'unidecode', 'mutagen', 'feedparser', \ + 'configobj'], + provides=['derailleur'], + platforms=['POSIX'], + classifiers=['Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: End Users/Desktop', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Natural Language :: English', + 'Operating System :: POSIX', + 'Programming Language :: Python :: 2.7', + 'Topic :: Internet'] + ) diff --git a/src/derailleur/__init__.py b/src/derailleur/__init__.py new file mode 100644 index 0000000..0b03267 --- /dev/null +++ b/src/derailleur/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Derailleur is a collection of tools to manage the Transmission client + and process the downloads once complete""" + +from .about import __version__ +from . import args +from . import config +from . import loggers +from . import client +from . import postproc +from . import history +from . import manager diff --git a/src/derailleur/about.py b/src/derailleur/about.py new file mode 100644 index 0000000..f3f856d --- /dev/null +++ b/src/derailleur/about.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Basic info""" + +__version__ = '0.31' +VERSION = __version__ +MAINTAINER = "Mads Michelsen " +DESCRIPTION = "A set of tools for the Transmission client" +URL = "https://github.com/brokkr/derailleur" diff --git a/src/derailleur/args/__init__.py b/src/derailleur/args/__init__.py new file mode 100644 index 0000000..182a19d --- /dev/null +++ b/src/derailleur/args/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Designation of basic arguments for the derailleur scripts""" + +import sys +import argparse +from derailleur import __version__ + + +def start_parser(): + '''Start a parser, leave open for custom arguments in script''' + about = 'Derailleur ' + __version__ + '\n' \ + + 'See wiki for help: https://github.com/brokkr/derailleur/wiki' + parser = argparse.ArgumentParser(description=about, formatter_class=\ + argparse.RawTextHelpFormatter) + parser.add_argument("-V", "--version", help="print version number", + action="store_true") + return parser + +def end_parser(parser): + '''Close up parser and return the user input''' + parsed_args = parser.parse_args() + if parsed_args.version: + print(str(__version__)) + sys.exit() + return parsed_args diff --git a/src/derailleur/client/__init__.py b/src/derailleur/client/__init__.py new file mode 100644 index 0000000..e244012 --- /dev/null +++ b/src/derailleur/client/__init__.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Establishing a connection to the Transmission client""" + +import urllib.request +import urllib.error +from os import path +from tempfile import TemporaryDirectory +import transmissionrpc + + +class TransmissionClient(transmissionrpc.Client): + '''TransmissionClient with added methods for downloading .torrents''' + def download(self, torrent_url, pause_when_added): + '''Validates and starts magnets, local and remote torrent files''' + if torrent_url[:6] == 'magnet': + pass + elif torrent_url[-8:] == '.torrent' and path.isfile(torrent_url): + torrent_url = "file://" + torrent_url + elif torrent_url[-8:] == '.torrent' and torrent_url[:4] == 'http': + tmp_dir = TemporaryDirectory() + file_path = path.join(tmp_dir.name, 'd.torrent') + try: + urllib.request.urlretrieve(torrent_url, file_path) + except urllib.error.HTTPError: + return False + torrent_url = "file://" + file_path + else: + return False + try: + torrent = self.add_torrent(torrent_url, paused=pause_when_added) + return torrent.name + except (transmissionrpc.error.TransmissionError, AttributeError): + return False + +def get_client(derailleurconfig): + '''Uses configobj to create a client''' + try: + transmissionclient = TransmissionClient( + derailleurconfig['Connection']['host'], + derailleurconfig['Connection']['port'], + derailleurconfig['Connection']['user'], + derailleurconfig['Connection']['password'] + ) + except transmissionrpc.error.TransmissionError: + transmissionclient = None + return transmissionclient diff --git a/src/derailleur/config/__init__.py b/src/derailleur/config/__init__.py new file mode 100644 index 0000000..f109afc --- /dev/null +++ b/src/derailleur/config/__init__.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Parses the configuration and validates it against an external schema""" + +import sys +import fileinput +from os import path, makedirs +from configobj import ConfigObj +from validate import Validator, ValidateError +from derailleur import __file__ as module_file + + +def nestring_check(value): + '''Requires string to not be empty or contain whitespace''' + if not isinstance(value, (str, int)): + raise ValidateError + if not value or ' ' in value: + raise ValidateError + return str(value) + +def path_check(value): + '''Very basic requirements for path string''' + if not isinstance(value, str): + raise ValidateError + if not value[0] == '/': + raise ValidateError + return str(value) + +def get_config(): + '''Create a configobj instance plus validation of user config''' + spec_file = path.join(path.dirname(module_file), 'config', + 'configspec.ini') + config_dir = path.join(path.expanduser('~'), '.derailleur') + config_path = path.join(config_dir, 'derailleur.conf') + if not path.isdir(config_dir): + makedirs(config_dir) + if not path.isfile(config_path): + config = ConfigObj(config_path, configspec=spec_file, stringify=False, + write_empty_values=True) + else: + config = ConfigObj(config_path, configspec=spec_file, stringify=True) + validator = Validator({'nestring': nestring_check, 'path': path_check}) + validation = config.validate(validator, copy=True) + if not path.isfile(config_path): + config.write() + print('New config file: ' + config_path) + sys.exit() + config.dir = config_dir + config.path = config_path + config.db = path.join(config_dir, 'derailleur.db') + config.log = { + 'Add': path.join(config_dir, 'derailleur-add.log'), + 'Feed': path.join(config_dir, 'derailleur-feed.log'), + 'Manager': path.join(config_dir, 'derailleur-manager.log'), + 'Postproc': path.join(config_dir, 'derailleur-postproc.log') + } + if validation is True: + validation = { + 'Connection': True, + 'Add': True, + 'Feed': True, + 'Manager': True, + 'Postproc': True, + 'Audio': True, + 'Smtp': True, + 'Kodi': True + } + return (config, validation) diff --git a/src/derailleur/config/configspec.ini b/src/derailleur/config/configspec.ini new file mode 100644 index 0000000..23c46d7 --- /dev/null +++ b/src/derailleur/config/configspec.ini @@ -0,0 +1,56 @@ +[Connection] +host = nestring(default=127.0.0.1) +port = integer(default=9091) +user = nestring(default=None) +password = nestring(default=None) + +[Add] +pause_when_added = boolean(default=false) +file_log = boolean(default=true) +mail_log = boolean(default=false) + +[Feed] +url = nestring(default=None) +pause_when_added = boolean(default=false) +file_log = boolean(default=true) +mail_log = boolean(default=false) + +[Manager] +hours_before_removal = integer(default=72) +delete_data = boolean(default=false) +seed_stops = list(default=None) +file_log = boolean(default=true) +mail_log = boolean(default=false) + +[Postproc] +tmp_path = path(default='/tmp') +base_path_tv = path(default='/tmp/derailleur/tv') +base_path_film = path(default='/tmp/derailleur/film') +base_path_music = path(default='/tmp/derailleur/music') +base_path_audiobook = path(default='/tmp/derailleur/audiobook') +base_path = path(default='/tmp/derailleur') +keywords = list(default=None) +file_log = boolean(default=true) +mail_log = boolean(default=false) + +[Audio] +retain_id3v1 = boolean(default=false) +retain_genre = boolean(default=false) +retain_cover_image = boolean(default=true) +rename_cover_image = nestring(default=cover) + +[Smtp] +mailhost = nestring(default=None) +mailport = integer(default=587) +fromaddr = nestring(default=None) +toaddr = nestring(default=None) +login = nestring(default=None) +password = nestring(default=None) + +[Kodi] +connect_to_kodi = boolean(default=false) +host = ip_addr(default=127.0.0.1) +port = integer(default=8080) +user = nestring(default=None) +password = nestring(default=None) + diff --git a/src/derailleur/history/__init__.py b/src/derailleur/history/__init__.py new file mode 100644 index 0000000..375100d --- /dev/null +++ b/src/derailleur/history/__init__.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Poca is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Jar exists only to save any object handed to it (although it defaults to + a dictionary). Likewise, once configured it will return said object when + called upon by the load method.""" + +import os +import pickle + + +class Jar: + '''Class for retrieving and saving feed/manager entry info''' + def __init__(self, config, db_filename): + self.db_filepath = os.path.join(config.dir, db_filename) + + def load(self): + '''Retrieve contents of pickle''' + if not os.path.isfile(self.db_filepath): + pickle_contents = {} + success = self.save(pickle_contents) + if not success: + return False + try: + with open(self.db_filepath, mode='rb') as f: + pickle_contents = pickle.load(f) + except (PermissionError, pickle.UnpicklingError, EOFError): + return False + return pickle_contents + + def save(self, pickle_contents): + '''Saves pickle_contents to file using pickle''' + try: + with open(self.db_filepath, 'wb') as f: + pickle.dump(pickle_contents, f) + return True + except (PermissionError, pickle.PicklingError, EOFError): + return False diff --git a/src/derailleur/loggers/__init__.py b/src/derailleur/loggers/__init__.py new file mode 100644 index 0000000..51263a6 --- /dev/null +++ b/src/derailleur/loggers/__init__.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""A module implmenting a subclassed version of the Logger class. Our version + pins some preferences to the instance and auto-adds any needed/requested + handlers, e.g. null, file and smtp.""" + +import logging +from logging import handlers + + +class Logger(logging.Logger): + '''Custom Logger class with auto-added handlers''' + def __init__(self, name, config, validation, level=logging.DEBUG, verbose=False): + super(self.__class__, self).__init__(name, level) + self.config = config + self.add_null_handler() + if self.config[self.name]['file_log']: + self.add_file_handler() + if self.config[self.name]['mail_log'] and validation['Smtp'] is True: + self.add_smtp_handler() + if verbose: + self.add_stream_handler() + + def add_null_handler(self): + '''Adding a basic null_handler''' + null_handler = logging.NullHandler() + self.addHandler(null_handler) + + def add_stream_handler(self): + '''Adding a basic stream_handler''' + stream_handler = logging.StreamHandler() + stream_handler.setLevel(logging.DEBUG) + self.addHandler(stream_handler) + + def add_file_handler(self): + '''Adding a filehandler''' + file_handler = logging.FileHandler(self.config.log[self.name]) + file_handler.setLevel(logging.INFO) + file_formatter = logging.Formatter("%(asctime)s %(message)s", + datefmt='%Y-%m-%d %H:%M') + file_handler.setFormatter(file_formatter) + self.addHandler(file_handler) + + def add_smtp_handler(self): + '''Adding an SMTP_handler''' + subject = 'derailleur-' + self.name.lower() + smtp_handler = handlers.SMTPHandler((self.config['Smtp']['mailhost'], + self.config['Smtp']['mailport']), + self.config['Smtp']['fromaddr'], + [self.config['Smtp']['toaddr']], + subject, + credentials=\ + (self.config['Smtp']['login'], + self.config['Smtp']['password']), + secure=()) + smtp_handler.setLevel(logging.INFO) + mail_formatter = logging.Formatter("%(message)s") + smtp_handler.setFormatter(mail_formatter) + self.addHandler(smtp_handler) diff --git a/src/derailleur/manager/__init__.py b/src/derailleur/manager/__init__.py new file mode 100644 index 0000000..a10bf9e --- /dev/null +++ b/src/derailleur/manager/__init__.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""A library for managing active and completed torrents""" + +from datetime import datetime +from collections import namedtuple + + +Incomplete = namedtuple('non_completed', 'all downloading paused') +SubSeeded = namedtuple('sub_ratio_seeded', 'all seeding paused') +SuperSeeded = namedtuple('ratio_seeded', 'all seeding paused') + +class Manager(): + '''Transmission session manager that sorts torrents according to status''' + def __init__(self, config, client, logger, jar): + self.client = client + self.config = config + self.logger = logger + self.jar = jar + torrents = client.get_torrents() + self.all = torrents + for torrent in torrents: + torrent.update() + torrent.private = 'Private' if torrent._fields['isPrivate'].value \ + else 'Public' + incomplete = [x for x in torrents if x.progress < 100] + incomplete_dl = [x for x in incomplete if x.status == 'downloading'] + incomplete_paused = [x for x in incomplete if x.status == 'stopped'] + self.incomplete = Incomplete(incomplete, + incomplete_dl, + incomplete_paused) + completed = [x for x in torrents if x.progress == 100] + sub_seeded = [x for x in completed if x.ratio < x.seed_ratio_limit] + sub_seeding = [x for x in sub_seeded if x.status == 'seeding'] + sub_paused = [x for x in sub_seeded if x.status == 'stopped'] + self.sub_seeded = SubSeeded(sub_seeded, + sub_seeding, + sub_paused) + super_seeded = [x for x in completed if x.ratio >= x.seed_ratio_limit] + super_seeding = [x for x in super_seeded if x.status == 'seeding'] + super_paused = [x for x in super_seeded if x.status == 'stopped'] + self.super_seeded = SuperSeeded(super_seeded, + super_seeding, + super_paused) + + def check_torrents(self): + '''Loop through various collections and check on them''' + self.logger.debug("=== DOWNLOADING ===") + self.logger.debug('{: <32.32s} type current'.format('')) + for torrent in self.incomplete.all: + self.print_downloading(torrent) + self.logger.debug('') + self.logger.debug("=== BELOW RATIO ===") + self.logger.debug('{: <32.32s} type current'.format('')) + for torrent in self.sub_seeded.all: + self.print_sub_seeded(torrent) + self.logger.debug('') + self.logger.debug("=== ABOVE RATIO: SEEDING ===") + self.logger.debug('{: <32.32s} type current last'.format('')) + for torrent in self.super_seeded.seeding: + self.check_super_seeding(torrent) + self.logger.debug('') + self.logger.debug("=== ABOVE RATIO: PAUSED ===") + self.logger.debug('{: <32.32s} type current hour ttl'.format('')) + for torrent in self.super_seeded.paused: + self.check_super_paused(torrent) + self.cleanup() + + def print_downloading(self, torrent): + '''Print info on dl torrents''' + stream = '{: <32.32s} {: <7.7s} {:3.2f}' + stream = stream.format(torrent.name, torrent.private, torrent.ratio) + self.logger.debug(stream) + + def print_sub_seeded(self, torrent): + '''Print info on seed torrents not yet at ratio''' + stream = '{: <32.32s} {: <7.7s} {:3.2f}' + stream = stream.format(torrent.name, torrent.private, torrent.ratio) + self.logger.debug(stream) + + def check_super_seeding(self, torrent): + '''Check up on torrents still seeding after reaching ratio''' + jar_dic = self.jar.load() + torrent_hash = torrent.hashString + if torrent_hash in jar_dic: + last_seed_stop = jar_dic[torrent_hash][1] + else: + last_seed_stop = torrent.seed_ratio_limit + # /stream logger + stream = '{: <32.32s} {: <7.7s} {:3.2f} {:3.2f}' + stream = stream.format(torrent.name, str(torrent.private), + torrent.ratio, last_seed_stop) + self.logger.debug(stream) + # stream logger/ + stop_tests = {seed_stop: last_seed_stop < float(seed_stop) \ + <= torrent.ratio for seed_stop in \ + self.config['Manager']['seed_stops']} + true_seed_stops = [x for x in stop_tests if stop_tests[x] is True] + if not true_seed_stops: + jar_dic[torrent_hash] = (None, last_seed_stop, torrent.name) + self.jar.save(jar_dic) + return + torrent.stop() + self.logger.info("We have stopped seeding %s @ ratio %0.2f." % + (torrent.name, torrent.ratio)) + jar_dic[torrent_hash] = (datetime.now(), torrent.ratio, torrent.name) + self.jar.save(jar_dic) + + def check_super_paused(self, torrent): + '''Check up on torrents that have paused after reaching ratio + (or have been paused because they've reached a seed stop)''' + jar_dic = self.jar.load() + torrent_hash = torrent.hashString + if not torrent_hash in jar_dic: + self.logger.info("%s has stopped seeding after reaching seed " + "limit %0.2f." % (torrent.name, torrent.ratio)) + jar_dic[torrent_hash] = (datetime.now(), torrent.ratio, torrent.name) + _success = self.jar.save(jar_dic) + return + paused_datetime = jar_dic[torrent_hash][0] + if paused_datetime: + time_stopped = datetime.now() - jar_dic[torrent_hash][0] + hours_paused = int(time_stopped.total_seconds() / 3600) + else: + hours_paused = 0 + jar_dic[torrent_hash] = (datetime.now(), torrent.ratio, torrent.name) + _success = self.jar.save(jar_dic) + wait_hours = int(self.config['Manager']['hours_before_removal']) + ttl = wait_hours - hours_paused + # /stream logger + stream = '{: <32.32s} {: <7.7s} {:3.2f} {:3.0f} {:3.0f}' + stream = stream.format(torrent.name, torrent.private, torrent.ratio, + hours_paused, ttl) + self.logger.debug(stream) + # stream logger/ + if hours_paused >= wait_hours: + self.client.remove_torrent(torrent.id, delete_data=\ + self.config['Manager']['delete_data']) + self.logger.info("%s has been removed after pausing for %i hours. " + "It seeded to %0.2f." + % (torrent.name, hours_paused, torrent.ratio)) + del(jar_dic[torrent_hash]) + _success = self.jar.save(jar_dic) + + def cleanup(self): + '''Cleanup routine that removes leftover records (e.g. for + entries that have been manually removed)''' + active_hashes = set([torrent.hashString for torrent in self.all]) + jar_dic = self.jar.load() + jar_hashes = set(jar_dic.keys()) + difference = jar_hashes.difference(active_hashes) + for torrent_hash in difference: + del(jar_dic[torrent_hash]) + _success = self.jar.save(jar_dic) diff --git a/src/derailleur/postproc/__init__.py b/src/derailleur/postproc/__init__.py new file mode 100644 index 0000000..fd872b8 --- /dev/null +++ b/src/derailleur/postproc/__init__.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Library for Transmission post-download sorting and filtering""" + +from . import audio +from . import film +from . import keywords +from . import stats +from . import tv diff --git a/src/derailleur/postproc/audio.py b/src/derailleur/postproc/audio.py new file mode 100644 index 0000000..e68880c --- /dev/null +++ b/src/derailleur/postproc/audio.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Submodule for processing audio files and their metadata""" + +import os +import re +import shutil +from collections import defaultdict + +from mutagen.easyid3 import EasyID3 +from mutagen.flac import FLAC +from mutagen.id3._util import ID3NoHeaderError + +from . import generic +from .functions import char_wash + + +META_FRAMES = ['artist', 'album', 'title', 'tracknumber', 'discnumber', + 'date', 'genre'] +EXT_CLASS_DIC = {'.mp3': EasyID3, '.flac': FLAC} + +class Torrent(generic.Torrent): + """Custom Torrent class for mp3 and flac files. Edits metadata and renames + according to scheme.""" + + ## 1: READING + + def read_tag(self, TagClass, tag_dic): + """run once per file, extracts metadata from tag using mutagen's + id3/flac common interface.""" + try: + tag = TagClass(tag_dic['filepath']) + tag_frames = list(set(META_FRAMES).intersection(list(tag.keys()))) + tag_metadata = {frame:tag[frame][0] for frame in tag_frames} + except ID3NoHeaderError: + tag_frames = [] + tag_metadata = {} + # remove '/' from track and disc numbers + fix_number = lambda x: int(str(x).split('/')[0]) + tag_numbers = {number: fix_number(tag_metadata[number]) for number + in ['tracknumber', 'discnumber'] if number + in tag_frames} + # merge all the metadata back into tag_dic + tag_metadata.update(tag_numbers) + tag_dic.update(tag_metadata) + tag_dic['frames'] = tag_frames + return tag_dic + + def variance(self, tag_name): + """Establish variance of tag values or missing values""" + try: + tag_set = set(tag_dic[tag_name] for tag_dic in self.tag_lst) + except KeyError: + tag_set = set() + return tag_set + + def read_tags(self): + """Creates a list of all music files with metadata information""" + self.tag_lst = [self.read_tag(EXT_CLASS_DIC[file_dic['ext']], file_dic) + for file_dic in self.stats.file_lst + if file_dic['ext'] in EXT_CLASS_DIC] + self.artist_no = len(self.variance('artist')) + self.album_no = len(self.variance('album')) + self.tracknumber_no = len(self.variance('tracknumber')) + self.title_no = len(self.variance('title')) + self.discnumber_no = len(self.variance('discnumber')) + self.track_width = len(str(max(self.variance('tracknumber')))) + + ## 2: NAMING SCHEME + + def rename_scheme(self, tag_dic): + """Work out the rename scheme based on number of artists and albums + in the total file mass""" + # level one + if self.artist_no == 0: + level_one = "z_missing_artist" + elif self.album_no == 0: + level_one = "z_missing_album" + elif self.artist_no == 1: + level_one = char_wash(tag_dic['artist']) + elif self.artist_no > 1: + level_one = "z_various artists" + # level two + if self.album_no == 0: + level_two = char_wash(tag_dic['parentdir']) + elif self.album_no > 0 and self.discnumber_no <= 1: + level_two = char_wash(tag_dic['album']) + elif self.album_no > 0 and self.discnumber_no > 1: + level_two = char_wash(tag_dic['album']) + '_disc_' + \ + str(tag_dic['discnumber']) + # file name + if self.tracknumber_no == 0 or self.title_no == 0: + filename = char_wash(tag_dic['basename']) + tag_dic['ext'] + elif self.album_no == 0: + filename = char_wash(tag_dic['basename']) + tag_dic['ext'] + elif self.artist_no <= 1: + filename = str(tag_dic['tracknumber']).zfill(self.track_width) + \ + '_' + char_wash(tag_dic['title']) + tag_dic['ext'] + elif self.artist_no > 1: + filename = str(tag_dic['tracknumber']).zfill(self.track_width) + \ + '_' + char_wash(tag_dic['artist']) + '_-_' + \ + char_wash(tag_dic['title']) + tag_dic['ext'] + # putting it all together + return os.path.join(self.type_path, level_one, level_two, filename) + + def rename(self): + '''Run rename scheme for each file''' + for tag_dic in self.tag_lst: + tag_dic['new_path'] = self.rename_scheme(tag_dic) + + ## 3: WRITING + + def new_tags(self): + """Create temp files, rewrite their tags, prepare for move_files""" + for tag_dic in self.tag_lst: + tmp_file_path = os.path.join(self.tmp_folder.name, + char_wash(tag_dic['filename'])) + shutil.copy(tag_dic['filepath'], tmp_file_path) + # run appropriate tag function on file to access and delete it + if tag_dic['frames']: + tag = EXT_CLASS_DIC[tag_dic['ext']](tmp_file_path) + tag.delete() + elif tag_dic['ext'] == '.mp3': + tag = EasyID3() + tag.save(tmp_file_path) + elif tag_dic['ext'] == '.flac': + tag = FLAC(tmp_file_path) + tag.save(tmp_file_path) + # why are we turning ints to strs? is it a requirement of mutagen? + for frame in tag_dic['frames']: + if frame == 'discnumber' and self.discnumber_no <= 1: + continue + if frame == 'genre' and \ + not self.config['Audio']['retain_genre']: + continue + if type(tag_dic[frame]) == int: + tag[frame] = [str(tag_dic[frame])] + else: + tag[frame] = [tag_dic[frame]] + tag.save(tmp_file_path) + if type(tag).__name__ == 'EasyID3' and \ + self.config['Audio']['retain_id3v1']: + tag.save(tmp_file_path, v1=2) + self.move_lst.append((tmp_file_path, tag_dic['new_path'])) + self.move_type_lst.append(tag_dic['filetype']) + + ## 4: COVERART + + def coverart(self): + """For every destination folder, choose an image if any. Available + images are scored for suitability by various factors, including + name, placement and size.""" + if not self.stats.file_by_type_dic['image']: + return + # for every destination, list origins + dest_dic = defaultdict(set) + for tag_dic in self.tag_lst: + ppath = os.path.dirname(tag_dic['new_path']) + dest_dic[ppath].add(tag_dic['folderpath']) + image_lst = self.stats.file_by_type_dic['image'] + max_image_size = max([image['bytesize'] for image in image_lst]) + # for every destination, list candidates + for dest_dir in list(dest_dic.keys()): + image_scores_dic = {} + for image in image_lst: + image_score = 0 + # image shares origin_folder with tags with this destination + if image['folderpath'] in dest_dic[dest_dir]: + if len(dest_dic[dest_dir]) == 1: + image_score += 1000 + else: + image_score += 500 + # basename is cover etc. or it's part of the basename + candidate_basenames = ['cover', 'front', 'folder', 'art'] + if image['basename'].lower() in candidate_basenames: + image_score += 200 + else: + regex = False + for basename in candidate_basenames: + re_test = re.compile('.*%s.*'%basename) + if re_test.match(image['basename'].lower()): + regex = True + if regex: + image_score += 100 + # give a bonus to the largest image (in bytesize) + size_score = float(image['bytesize']) / max_image_size * 99 + image_score += size_score + # enter the image and the image score into the dictionary + image_path = os.path.join(image['folderpath'], + image['filename']) + image_scores_dic[image_path] = image_score + # find the image with the highest score for that destination + highscorer = max(iter(image_scores_dic.keys()), + key=(lambda key: image_scores_dic[key])) + # Add winner to move_lst + new_image_name = self.config['Audio']['rename_cover_image'] + \ + os.path.splitext(highscorer)[1] + new_file_path = os.path.join(self.type_path, dest_dir, + new_image_name) + self.move_lst.append((highscorer, new_file_path)) + self.move_type_lst.append('image') + + + def manipulate(self): + """Overall music manipulation function, encompassing read_tags, + categorize, rename, new_tags, and coverart.""" + self.read_tags() + self.rename() + self.new_tags() + if self.config['Audio']['retain_cover_image']: + self.coverart() diff --git a/src/derailleur/postproc/film.py b/src/derailleur/postproc/film.py new file mode 100644 index 0000000..8c3fa03 --- /dev/null +++ b/src/derailleur/postproc/film.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Submodule for processing movie files""" + +import os +import re +from . import generic +from .functions import char_rinse + + +SOURCES = ['webdl', 'web-dl', 'brrip', 'bluray', 'hdtv', 'dvdrip'] +RESOLUTIONS = ['1080p', '720p'] +VIDEO = ['h264', 'x264', 'xvid', 'divx'] +AUDIO = ['dts', 'aac2', 'ac3', 'mp3', 'aac'] +CHANNELS = ['5 1', '6ch'] + +class Torrent(generic.Torrent): + """Film content specific Torrent. Filters files and creates containing + folder according to naming scheme.""" + + def manipulate(self): + """Use regex to extract information from name, then reconstruct + according to scheme""" + # lowercase, replace points by spaces, remove multiple spaces & parantheses + if os.path.isfile(self.torrent_path): + tname = os.path.splitext(self.torrent_name)[0] + else: + tname = self.torrent_name + tname = re.sub('[\[\]\.()]', ' ', tname) + tname = re.sub(' +', ' ', tname.lower()) + # regex searches for year and resolution info + re_year = re.search(r'(19[3-9]\d|20[0-1]\d)', tname) + re_resolution = re.search(r'1080p|720p', tname) + # construct folder name according to scheme + if re_year: + dir_name = tname[0:re_year.start()] + '(' + re_year.group() + ')' + else: + dir_name = tname + if re_resolution: + dir_name = dir_name + ' [' + re_resolution.group() + ']' + dir_name = char_rinse(dir_name) + # get video files and subtitles, ignore the rest, apply filters + keep_types = ['video', 'subtitle'] + keepers = [f for f in self.stats.file_lst if f['filetype'] in keep_types] + for file_dic in keepers: + if re.search(r'sample', file_dic['filename'].lower()): + continue + if file_dic['filetype'] == 'video' and file_dic['bytesize'] < 100000000: + continue + old_file_path = file_dic['filepath'] + new_file_path = os.path.join(self.type_path, + dir_name, file_dic['filename']) + self.move_lst.append((old_file_path, new_file_path)) + self.move_type_lst.append(file_dic['filetype']) diff --git a/src/derailleur/postproc/functions.py b/src/derailleur/postproc/functions.py new file mode 100644 index 0000000..9953231 --- /dev/null +++ b/src/derailleur/postproc/functions.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""A collection of string manipulation functions, mostly transliteration""" + +import string +from unidecode import unidecode + + +## Filter rules +WHITESTR = string.ascii_lowercase + string.digits + '_' + '-' +DANISH = [('Æ', 'Ae'), ('Ø', 'Oe'), ('Å', 'Aa'), ('æ', 'ae'), ('ø', 'oe'), + ('å', 'aa')] +REPLACEMENTS = [('&', ' and '), ('=', ' equal '), ('+', ' plus '), + ('/', ' or '), ('@', ' at ')] + +## Functions +def asciify(_str): + '''Transliterates unicode into ascii''' + ustr = str(_str) + for swap in DANISH: + ustr = ustr.replace(swap[0], swap[1]) + astr = unidecode(ustr) + return astr + +def char_rinse(orgstring): + '''Custom replacements''' + rstr = asciify(orgstring) + # replace spaces with underscores + special cases: + # lowercase + for swap in REPLACEMENTS: + rstr = orgstring.replace(swap[0], swap[1]) + rstr = rstr.lower() + return rstr + +def char_wash(orgstring): + '''Replace whitespace''' + wstr = char_rinse(orgstring) + # underscores and filter anything not asciiish remaining + wstr = wstr.replace(' ', '_') + whitefilter = lambda x: x in WHITESTR + wstr = ''.join(list(filter(whitefilter, wstr))) + # remove multiple underscores + while '__' in wstr: + wstr = wstr.replace('__', '_') + # remove underscores from end of filename + if len(wstr) == 0: + wstr = '_empty' + if len(wstr) > 1: + if wstr[-1] == '_': + wstr = wstr[0:-1] + return wstr diff --git a/src/derailleur/postproc/generic.py b/src/derailleur/postproc/generic.py new file mode 100644 index 0000000..56c6b0f --- /dev/null +++ b/src/derailleur/postproc/generic.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Defining the standard torrent class""" + +import os +import shutil +from tempfile import TemporaryDirectory + + +class Torrent: + """Basic torrent class with all required values and functions. Exists as + template for more customized content specific Torrent classes.""" + + def __init__(self, config, stats, type_path): + """Creates instance of class. Universal, never overridden.""" + self.config = config + self.stats = stats + self.type_path = type_path + self.torrent_name = stats.torrent_name + self.torrent_path = stats.torrent_path + self.torrent_ppath = stats.torrent_ppath + self.move_lst, self.move_type_lst = [], [] + self.tmp_folder = config['Postproc']['tmp_path'] + + def manipulate(self): + """Changes files and variables before selecting the relevant files for + move_files. Overridden by all but default content types.""" + for file_dic in self.stats.file_lst: + rel_file_path = os.path.relpath(file_dic['filepath'], + self.torrent_ppath) + new_file_path = os.path.join(self.type_path, rel_file_path) + self.move_lst.append((file_dic['filepath'], new_file_path)) + self.move_type_lst.append(file_dic['filetype']) + + def move_files(self): + """Copies each individual file (no folders) selected from origin + to chosen destination. Universal, never overridden.""" + errors = False + for file_tuple in self.move_lst: + new_ppath = os.path.dirname(file_tuple[1]) + if not os.path.isdir(new_ppath): + try: + os.makedirs(new_ppath) + except (PermissionError, OSError, IOError): + errors = True + continue + try: + shutil.copy(file_tuple[0], file_tuple[1]) + except IOError: + errors = True + self.tmp_folder.cleanup() + return errors diff --git a/src/derailleur/postproc/keywords.py b/src/derailleur/postproc/keywords.py new file mode 100644 index 0000000..d357389 --- /dev/null +++ b/src/derailleur/postproc/keywords.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Defining the keywords specific torrent class""" + +import os +from . import generic + + +class Torrent(generic.Torrent): + """Keyword content type""" + + def manipulate(self): + """Changes files and variables before selecting the relevant files for + move_files. Keyword special that removes keyword from file name.""" + for file_dic in self.stats.file_lst: + keyword = self.stats.identity + rel_file_path = os.path.relpath(file_dic['filepath'], + self.torrent_ppath) + rel_file_path = rel_file_path[len(keyword)+1:] + new_file_path = os.path.join(self.type_path, rel_file_path) + self.move_lst.append((file_dic['filepath'], new_file_path)) + self.move_type_lst.append(file_dic['filetype']) diff --git a/src/derailleur/postproc/stats.py b/src/derailleur/postproc/stats.py new file mode 100644 index 0000000..4a4f4b3 --- /dev/null +++ b/src/derailleur/postproc/stats.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Defining the stats class""" + +import os +import re + + +class Stats: + """File stats useful for identification""" + + def create_file_dic(self, folder_path, f): + """Run once per file, returns all general information about file""" + file_dic = {} + file_dic['filename'] = f + file_dic['folderpath'] = folder_path + file_dic['parentdir'] = os.path.split(folder_path)[1] + file_dic['filepath'] = os.path.join(folder_path, f) + file_dic['basename'], file_dic['ext'] = os.path.splitext(f) + file_dic['bytesize'] = os.path.getsize(file_dic['filepath']) + file_type = [k for k, v in list(self.ext_dic.items()) \ + if file_dic['ext'] in v] + file_dic['filetype'] = ''.join(file_type) + return file_dic + + def identify(self, keywords): + """Analyzes naming and file stats to determine category of files""" + self.identity = 'generic' + for keyword in keywords: + re_keyword = re.search('^' + keyword + ' ', self.torrent_name) + if re_keyword: + self.identity = keyword + return + if self.torrent_name[0:10] == 'audiobook ': + self.identity = 'audiobook' + elif self.type_size_pct['music'] > 80: + self.identity = 'music' + elif self.type_size_pct['video'] > 80: + # tv torrent names either contain the season or season and episode + re_tv = re.compile('(?i).*season.*|.*series.*|.*complete.*|' + '.*s\d{1,2}.*|.*s\d{1,2}e\d{1,2}.*|' + '.*(\d{1,2})x(\d\d)') + re_daily = re.compile('.*(20[0-2][0-9]\.[0-1][0-9]\.[0-3][0-9]).*') + # films torrent names (almost) always contain a year (1930-2019) + re_film = re.compile('.*(19[3-9][0-9]|20[0-2][0-9]).*') + if re_tv.match(self.torrent_name): + self.identity = 'tv' + elif re_daily.match(self.torrent_name): + self.identity = 'tv' + elif re_film.match(self.torrent_name): + self.identity = 'film' + + def __init__(self, keywords, torrent_info): + """Creates instance of Stats class""" + self.torrent_name = torrent_info.name + self.torrent_ppath = torrent_info.ppath + self.torrent_path = os.path.join(torrent_info.ppath, torrent_info.name) + # what types are various file extensions + self.ext_dic = { + 'package': ['.apk', '.deb', '.rpm'], + 'music': ['.mp3', '.ogg', '.flac', '.wma', '.m4a'], + 'video': ['.avi', '.mkv', '.wmv', '.mp4', '.3gp'], + 'text': ['.txt', '.nfo', '.sfv', '.idx', '.url', '.m3u'], + 'book': ['.pdf', '.epub', '.mobi'], + 'subtitle': ['.srt', '.smi', '.sub'], + 'image': ['.jpg', '.jpeg', '.png', '.gif', '.bmp'], + 'compressed': ['.zip', '.rar', '.gz'], + 'disc': ['.iso'], + 'part': ['.part'] + } + for i in range(0, 100): + self.ext_dic['compressed'].append(".r%02d" % i) + # initialize containers + self.file_lst, self.file_by_type_dic = [], {} + self.type_count, self.type_size = {}, {} + self.type_count_pct, self.type_size_pct = {}, {} + # tree walking + if os.path.isfile(self.torrent_path): + file_dic = self.create_file_dic(torrent_info.ppath, torrent_info.name) + self.file_lst.append(file_dic) + else: + for (root, dirs, files) in os.walk(self.torrent_path): + dir_contents = [self.create_file_dic(root, f) for f in files] + self.file_lst.extend(dir_contents) + # discount .part files + self.file_lst = [file_dic for file_dic in self.file_lst if + file_dic['filetype'] != 'part'] + # calculate overall torrent stats + self.torrent_count = len(self.file_lst) + self.torrent_size = sum([fdic['bytesize'] for fdic in self.file_lst]) + if not self.torrent_count: + self.torrent_count = 1 + if not self.torrent_size: + self.torrent_size = 1 + # calculate stats for represented file_types + for file_type in self.ext_dic: + file_type_lst = [file_dic for file_dic in self.file_lst \ + if file_dic['filetype'] == file_type] + self.file_by_type_dic[file_type] = file_type_lst + self.type_count[file_type] = len(file_type_lst) + self.type_size[file_type] = sum([file_dic['bytesize'] \ + for file_dic in file_type_lst]) + self.type_count_pct[file_type] = float(self.type_count[file_type]) \ + / float(self.torrent_count) * 100 + self.type_size_pct[file_type] = float(self.type_size[file_type]) \ + / float(self.torrent_size) * 100 + + self.identify(keywords) diff --git a/src/derailleur/postproc/tv.py b/src/derailleur/postproc/tv.py new file mode 100644 index 0000000..5b8da39 --- /dev/null +++ b/src/derailleur/postproc/tv.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Defining the TV class""" + +import os +import re +from . import generic + + +class Torrent(generic.Torrent): + """TV specific content type. Establishes series name/season number + for kodi friendly folder scheme (series name/season no/files). + Works for series, seasons and individual episodes alike.""" + + def manipulate(self): + """Use regexes to find series name and season number, create folder + scheme, filter junk files (anything but 'real' video and subs)""" + # lowercase, replace points by spaces, remove multiple occurences + tname = self.torrent_name.lower().replace(".", " ").replace("_", " ") + tname = re.sub(' +', ' ', tname) + # find indices for common title termination indicators + re_sxex = re.search(r's(\d\d)e(\d\d)|(\d{1,2})x(\d\d)', tname) + re_season = re.search(r'season (\d+)|series (\d+)|s(\d\d)', tname) + re_complete = re.search(r'complete', tname) + re_year = re.search(r'(19[3-9]\d|20[0-1]\d)', tname) + searches = [re_sxex, re_season, re_complete, re_year] + re_indices = [result.start() for result in searches if result] + # use lowest index to get title + if not re_indices: + re_indices = [-1] + lowest_index = min(re_indices) + series_name = tname[0:lowest_index] + # clean up series name (allow closing parenthesis for '(US)' etc.) + self.series_name = re.match(r'.*[^\-\_\ \(]', series_name).group() + # get fallback data from torrent name and set series/episode destination + if re_sxex: + se = [int(result) for result in re_sxex.groups() if result] + season_no, episode_no = se[0], se[1] + type_path = os.path.join(self.type_path, 'episodes') + elif re_season: + season_no = [int(result) for result in re_season.groups() if result][0] + type_path = os.path.join(self.type_path, 'series') + else: + season_no, episode_no = (None, None) + type_path = os.path.join(self.type_path, 'series') + # get video files and subtitles, ignore the rest + keep_types = ['video', 'subtitles'] + keepers = [f for f in self.stats.file_lst if f['filetype'] in keep_types] + for file_dic in keepers: + file_name = file_dic['filename'].lower() + if re.search(r'sample', file_name): + continue + old_file_path = file_dic['filepath'] + new_folder_path = os.path.join(type_path, self.series_name) + # override season_no on a per-episode basis + re_f_sxex = re.search(r's(\d\d)e(\d\d)', file_name) + re_f_exe = re.search(r'(\d\d)x(\d\d)', file_name) + if re_f_exe: + season_no = int(re_f_exe.group(1)) + if re_f_sxex: + season_no = int(re_f_sxex.group(1)) + if season_no: + season = 'season ' + str(season_no) + new_folder_path = os.path.join(new_folder_path, season) + new_file_path = os.path.join(new_folder_path, file_dic['filename']) + self.move_lst.append((old_file_path, new_file_path)) + self.move_type_lst.append(file_dic['filetype']) diff --git a/src/scripts/derailleur-add b/src/scripts/derailleur-add new file mode 100755 index 0000000..d6473aa --- /dev/null +++ b/src/scripts/derailleur-add @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Torrent starter""" + +import sys +import derailleur + + +def get_parsed_args(): + '''Parses arguments and return them''' + parser = derailleur.args.start_parser() + parser.add_argument("torrent", nargs='*', + help="Magnet URL(s) or .torrent(s)") + parsed_args = derailleur.args.end_parser(parser) + return parsed_args + +def start_torrent(config, client, logger, torrent): + '''For each torrent in nargs, hand it over''' + outcome = client.download(torrent, config['Add']['pause_when_added']) + if outcome: + logger.info(outcome) + else: + logger.error('ERROR : ' + 'Failed to add torrent') + +def main(): + '''Putting it all together''' + args = get_parsed_args() + config, validation = derailleur.config.get_config() + if validation['Connection'] is not True or validation['Add'] is not True: + sys.exit('ERROR : Missing configuration') + logger = derailleur.loggers.Logger('Add', config, validation) + client = derailleur.client.get_client(config) + if not client: + logger.error('ERROR : Connect to Transmission failed') + sys.exit(1) + for torrent in args.torrent: + start_torrent(config, client, logger, torrent) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + pass diff --git a/src/scripts/derailleur-feed b/src/scripts/derailleur-feed new file mode 100755 index 0000000..fd8d47c --- /dev/null +++ b/src/scripts/derailleur-feed @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +import sys +import feedparser +import derailleur + + +def get_parsed_args(): + '''Parses arguments and return them''' + parser = derailleur.args.start_parser() + parsed_args = derailleur.args.end_parser(parser) + return parsed_args + +def get_feed(config): + '''Parses feed and returns dictionary of entries''' + doc = feedparser.parse(config['Feed']['url']) + get_edic = lambda x: {'title': x.title, 'url': x.link, 'done': False} + feed_dic = {exml.id: get_edic(exml) for exml in doc.entries} + return feed_dic + +def get_fresh(feed_dic, jar_dic): + '''Go through feed entries''' + done_entries = [eid for eid in jar_dic if jar_dic[eid]['done'] is True] + fresh_entries = [eid for eid in feed_dic if eid not in done_entries] + return fresh_entries + +def start_torrent(config, client, logger, feed_dic, eid): + '''Start an entry from the feed''' + edic = feed_dic[eid] + outcome = client.download(edic['url'], config['Feed']['pause_when_added']) + if outcome: + edic['done'] = True + logger.info(outcome) + else: + logger.error('ERROR : Failed to add %s', edic['title']) + return edic + +def main(): + '''Putting it all together''' + _args = get_parsed_args() + config, validation = derailleur.config.get_config() + # note that defaults should ensure always-valid configs but bad user + # input (e.g. a string for a port number) may invalidate it. + if validation['Connection'] is not True or validation['Feed'] is not True: + sys.exit('ERROR : Missing configuration') + logger = derailleur.loggers.Logger('Feed', config, validation) + client = derailleur.client.get_client(config) + if not client: + logger.error('ERROR : Connect to Transmission failed') + sys.exit(1) + feed_dic = get_feed(config) + if not feed_dic: + logger.error('ERROR : Cannot find feed or feed empty') + sys.exit(1) + jar = derailleur.history.Jar(config, 'derailleur-feed.db') + jar_dic = jar.load() + if jar_dic is False: + logger.error('ERROR : Failure loading %s', jar.db_filepath) + sys.exit(1) + fresh_entries = get_fresh(feed_dic, jar_dic) + for eid in fresh_entries: + edic = start_torrent(config, client, logger, feed_dic, eid) + jar_dic[eid] = edic + success = jar.save(jar_dic) + if success is False: + logger.error('ERROR : Failure saving %s', jar.db_filepath) + sys.exit(1) + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + pass diff --git a/src/scripts/derailleur-manager b/src/scripts/derailleur-manager new file mode 100755 index 0000000..261ee37 --- /dev/null +++ b/src/scripts/derailleur-manager @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +"""Torrent manager script""" + +import sys +import derailleur + + +def get_parsed_args(): + '''Parses arguments and return them''' + parser = derailleur.args.start_parser() + parser.add_argument("-v", "--verbose", help="Get info on current torrents", + action="store_true") + parsed_args = derailleur.args.end_parser(parser) + return parsed_args + +def main(): + '''Putting it all together''' + args = get_parsed_args() + config, validation = derailleur.config.get_config() + if validation['Connection'] is not True or \ + validation['Manager'] is not True: + sys.exit('ERROR : Missing configuration') + logger = derailleur.loggers.Logger('Manager', config, validation, + verbose=args.verbose) + client = derailleur.client.get_client(config) + if not client: + logger.error('ERROR : Connect to Transmission failed') + sys.exit(1) + jar = derailleur.history.Jar(config, 'derailleur-manager.db') + manager = derailleur.manager.Manager(config, client, logger, jar) + manager.check_torrents() + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + pass diff --git a/src/scripts/derailleur-postproc b/src/scripts/derailleur-postproc new file mode 100755 index 0000000..e25e3c8 --- /dev/null +++ b/src/scripts/derailleur-postproc @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2016-2017 Mads Michelsen (mail@brokkr.net) +# This file is part of Derailleur. +# Derailleur is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +import os +import sys +from collections import namedtuple +import derailleur + +try: + from xbmcjson import XBMC + import requests.packages.urllib3.exceptions +except ImportError: + XBMC = None + +TorrentInfo = namedtuple('torrent_info', 'ppath name hash') + +def get_parsed_args(): + '''Parses arguments and return them''' + parser = derailleur.args.start_parser() + parser.add_argument("-t", "--target", help='''Set a target manually ''' + '''rather than relying on environmental variables''') + parsed_args = derailleur.args.end_parser(parser) + return parsed_args + +def get_torrent_info(args): + '''Get Transmission info on target''' + if args.target: + target = os.path.abspath(args.target) + if not os.path.isdir(target) and not os.path.isfile(target): + sys.exit('ERROR : Not a valid target') + get_pp = lambda x: os.path.abspath(os.path.join(x, os.path.pardir)) + torrent_info = TorrentInfo(ppath=get_pp(target), + name=os.path.basename(target), + hash='abcdef123456') + else: + try: + torrent_info = TorrentInfo(ppath=os.environ['TR_TORRENT_DIR'], + name=os.environ['TR_TORRENT_NAME'], + hash=os.environ['TR_TORRENT_HASH']) + except KeyError: + torrent_info = None + return torrent_info + +def get_type(config, keywords, stats): + '''Get torrent type''' + module_dic = { + 'tv': derailleur.postproc.tv, + 'film': derailleur.postproc.film, + 'music': derailleur.postproc.audio, + 'audiobook': derailleur.postproc.audio + } + if stats.identity in keywords: + type_module = derailleur.postproc.keywords + type_path = os.path.join(config['Postproc']['base_path'], + stats.identity) + elif stats.identity == 'generic': + type_module = derailleur.postproc.generic + type_path = os.path.join(config['Postproc']['base_path'], 'default') + else: + type_module = module_dic[stats.identity] + type_path = config['Postproc']['base_path_' + stats.identity] + return (type_module, type_path) + +def get_log_str(torrent, stats, move_errors): + '''Logging''' + log_str = "ERRORS ENCOUNTERED IN FILE OPERATION. " if move_errors else "" + log_str += "NAME: " + stats.torrent_name + ". TYPE: " + stats.identity + log_str += ". FILES: " + type_count = [x + " " + str(torrent.move_type_lst.count(x)) + for x in set(torrent.move_type_lst)] + log_str += ", ".join(type_count) + common_path = os.path.commonprefix([x[1] for x in torrent.move_lst]) + log_str += ". PARENT: " + os.path.dirname(common_path) + return log_str + +def update_kodi(config, stats): + '''Update kodi library''' + url = 'http://' + config['Kodi']['host'] + ':' \ + + str(config['Kodi']['port']) + '/jsonrpc' + try: + kodi = XBMC(url, config['Kodi']['user'], config['Kodi']['password']) + except requests.packages.urllib3.exceptions.NewConnectionError: + sys.exit(11) + if stats.identity in ['tv', 'film']: + kodi.VideoLibrary.Scan() + if stats.identity in ['music']: + kodi.AudioLibrary.Scan() + +def main(): + '''Putting the whole thing together''' + args = get_parsed_args() + config, validation = derailleur.config.get_config() + if validation['Postproc'] != True: + sys.exit('ERROR : Config not valid') + logger = derailleur.loggers.Logger('Postproc', config, validation) + torrent_info = get_torrent_info(args) + if not torrent_info: + logger.error('ERROR : Could not access Transmission variables') + sys.exit(1) + keywords = config['Postproc']['keywords'] + stats = derailleur.postproc.stats.Stats(keywords, torrent_info) + type_module, type_path = get_type(config, keywords, stats) + torrent = type_module.Torrent(config, stats, type_path) + torrent.manipulate() + move_errors = torrent.move_files() + log_str = get_log_str(torrent, stats, move_errors) + logger.info(log_str) + if XBMC and config['Kodi']['connect_to_kodi']: + update_kodi(config, stats) + sys.exit(0) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + pass