diff --git a/.gitignore b/.gitignore index 6342047..dd2c92d 100644 --- a/.gitignore +++ b/.gitignore @@ -169,4 +169,11 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ + +# Local agent/editor state +.continue/ +temp/ +.envs/.local/.django +start-dev.sh +opencode.json diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 55509fe..0000000 --- a/.pylintrc +++ /dev/null @@ -1,14 +0,0 @@ -[MASTER] -load-plugins=pylint_django, pylint_celery -django-settings-module=config.settings.base -[FORMAT] -max-line-length=120 - -[MESSAGES CONTROL] -disable=missing-docstring,invalid-name - -[DESIGN] -max-parents=13 - -[TYPECHECK] -generated-members=REQUEST,acl_users,aq_parent,"[a-zA-Z]+_set{1,2}",save,delete diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index b4cf0c0..0000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 - -sphinx: - configuration: docs/conf.py - -build: - image: testing - -python: - version: 3.9 - install: - - requirements: requirements/local.txt diff --git a/COPYING b/COPYING deleted file mode 100644 index 94a9ed0..0000000 --- a/COPYING +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/article/__init__.py b/article/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/article/admin.py b/article/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/article/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/article/apps.py b/article/apps.py deleted file mode 100644 index 8c0e2c9..0000000 --- a/article/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class ArticleConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - name = "article" diff --git a/article/management/__init__.py b/article/management/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/article/management/commands/__init__.py b/article/management/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/article/management/commands/load_articles_by_year.py b/article/management/commands/load_articles_by_year.py deleted file mode 100644 index 335598e..0000000 --- a/article/management/commands/load_articles_by_year.py +++ /dev/null @@ -1,80 +0,0 @@ -from django.core.management.base import BaseCommand - -from article.tasks import task_load_article_from_opac, task_load_article_from_article_meta - - -class Command(BaseCommand): - help = 'Generate task requests for loading article data from Article Meta for each year from 1900 to 2025' - - def add_arguments(self, parser): - parser.add_argument( - '--start-year', - type=int, - default=1990, - help='Start year (default: 1990)' - ) - parser.add_argument( - '--end-year', - type=int, - default=2025, - help='End year (default: 2025)' - ) - parser.add_argument( - '--collection', - type=str, - default='scl', - help='Collection code (default: scl)' - ) - parser.add_argument( - '--task', - choices=['load_article_from_opac', 'load_article_from_article_meta'], - default='load_article_from_opac', - help='Task to execute (default: load_article_from_opac)', - ) - - def handle(self, *args, **options): - start_year = options['start_year'] - end_year = options['end_year'] - collection = options['collection'] - - self.stdout.write( - self.style.SUCCESS( - f'Generating task requests from {start_year} to {end_year} for collection: {collection}' - ) - ) - - total_tasks = 0 - - for year in range(start_year, end_year + 1): - from_date = f'{year}-01-01' - until_date = f'{year}-12-31' - - self.stdout.write(f'Queuing task for year {year}...') - - # Queue the task for each year - if options['task'] == 'load_article_from_article_meta': - task_result = task_load_article_from_article_meta.delay( - from_date=from_date, - until_date=until_date, - collection=collection - ) - else: - task_result = task_load_article_from_opac.delay( - from_date=from_date, - until_date=until_date, - collection=collection - ) - - total_tasks += 1 - - self.stdout.write( - self.style.SUCCESS( - f'✓ Task queued for year {year}: {from_date} to {until_date} (Task ID: {task_result.id})' - ) - ) - - self.stdout.write( - self.style.SUCCESS( - f'\nCompleted! {total_tasks} tasks have been queued successfully.' - ) - ) diff --git a/article/migrations/0001_initial.py b/article/migrations/0001_initial.py deleted file mode 100644 index 816d61e..0000000 --- a/article/migrations/0001_initial.py +++ /dev/null @@ -1,137 +0,0 @@ -# Generated by Django 5.0.7 on 2025-02-07 17:50 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - ("collection", "0001_initial"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name="Article", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "created", - models.DateTimeField( - auto_now_add=True, verbose_name="Creation date" - ), - ), - ( - "updated", - models.DateTimeField( - auto_now=True, verbose_name="Last update date" - ), - ), - ( - "scielo_issn", - models.CharField( - db_index=True, max_length=9, verbose_name="SciELO ISSN" - ), - ), - ( - "pid_v2", - models.CharField( - db_index=True, max_length=23, verbose_name="PID V2" - ), - ), - ( - "pid_v3", - models.CharField( - blank=True, - db_index=True, - max_length=23, - null=True, - verbose_name="PID V3", - ), - ), - ( - "pdfs", - models.JSONField( - blank=True, - default=dict, - null=True, - verbose_name="Format with Language", - ), - ), - ( - "default_lang", - models.CharField(max_length=2, verbose_name="Default Language"), - ), - ( - "text_langs", - models.JSONField( - blank=True, - default=dict, - null=True, - verbose_name="Text Languages", - ), - ), - ( - "processing_date", - models.CharField(max_length=32, verbose_name="Processing Date"), - ), - ( - "publication_date", - models.CharField(max_length=32, verbose_name="Publication Date"), - ), - ( - "publication_year", - models.CharField( - db_index=True, max_length=4, verbose_name="Publication Year" - ), - ), - ( - "collection", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="collection.collection", - verbose_name="Collection", - ), - ), - ( - "creator", - models.ForeignKey( - editable=False, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="%(class)s_creator", - to=settings.AUTH_USER_MODEL, - verbose_name="Creator", - ), - ), - ( - "updated_by", - models.ForeignKey( - blank=True, - editable=False, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="%(class)s_last_mod_user", - to=settings.AUTH_USER_MODEL, - verbose_name="Updater", - ), - ), - ], - options={ - "verbose_name": "Article", - "verbose_name_plural": "Articles", - "unique_together": {("collection", "scielo_issn", "pid_v2", "pid_v3")}, - }, - ), - ] diff --git a/article/migrations/0002_alter_article_unique_together_article_files_and_more.py b/article/migrations/0002_alter_article_unique_together_article_files_and_more.py deleted file mode 100644 index cee055c..0000000 --- a/article/migrations/0002_alter_article_unique_together_article_files_and_more.py +++ /dev/null @@ -1,42 +0,0 @@ -# Generated by Django 5.0.7 on 2025-04-01 01:09 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("article", "0001_initial"), - ("collection", "0001_initial"), - ] - - operations = [ - migrations.AddField( - model_name="article", - name="files", - field=models.JSONField( - blank=True, default=dict, null=True, verbose_name="Files" - ), - ), - migrations.AddField( - model_name="article", - name="pid_generic", - field=models.CharField( - blank=True, - db_index=True, - max_length=50, - null=True, - verbose_name="PID Generic", - ), - ), - migrations.RemoveField( - model_name="article", - name="pdfs", - ), - migrations.AlterUniqueTogether( - name="article", - unique_together={ - ("collection", "scielo_issn", "pid_v2", "pid_v3", "pid_generic") - }, - ), - - ] diff --git a/article/migrations/0003_article_collection_scielo_issn_idx.py b/article/migrations/0003_article_collection_scielo_issn_idx.py deleted file mode 100644 index 753ac98..0000000 --- a/article/migrations/0003_article_collection_scielo_issn_idx.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 5.0.7 on 2025-06-12 17:16 - -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("article", "0002_alter_article_unique_together_article_files_and_more"), - ("collection", "0001_initial"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.AddIndex( - model_name="article", - index=models.Index( - fields=["collection", "scielo_issn"], name="collection_scielo_issn_idx" - ), - ), - ] diff --git a/article/migrations/__init__.py b/article/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/article/models.py b/article/models.py deleted file mode 100644 index 80d2a97..0000000 --- a/article/models.py +++ /dev/null @@ -1,143 +0,0 @@ -from django.db import models -from django.utils.translation import gettext_lazy as _ - -from core.models import CommonControlField -from collection.models import Collection - - -class Article(CommonControlField): - collection = models.ForeignKey( - Collection, - verbose_name=_('Collection'), - on_delete=models.CASCADE, - blank=False, - null=False, - db_index=True, - ) - - scielo_issn = models.CharField( - verbose_name=_('SciELO ISSN'), - max_length=9, - blank=False, - null=False, - db_index=True, - ) - - pid_v2 = models.CharField( - verbose_name=_('PID V2'), - max_length=23, - blank=False, - null=False, - db_index=True, - ) - - pid_v3 = models.CharField( - verbose_name=_('PID V3'), - max_length=23, - blank=True, - null=True, - db_index=True, - ) - - pid_generic = models.CharField( - verbose_name=_('PID Generic'), - max_length=50, - blank=True, - null=True, - db_index=True, - ) - - files = models.JSONField( - verbose_name=_('Files'), - null=True, - blank=True, - default=dict, - ) - - default_lang = models.CharField( - verbose_name=_('Default Language'), - max_length=2, - blank=False, - null=False, - ) - - text_langs = models.JSONField( - verbose_name=_('Text Languages'), - null=True, - blank=True, - default=dict, - ) - - processing_date = models.CharField( - verbose_name=_('Processing Date'), - max_length=32, - null=False, - blank=False, - ) - - publication_date = models.CharField( - verbose_name=_('Publication Date'), - max_length=32, - null=False, - blank=False, - ) - - publication_year = models.CharField( - verbose_name=_('Publication Year'), - max_length=4, - null=False, - blank=False, - db_index=True, - ) - - def __str__(self): - return f'{self.collection.acron3} - {self.scielo_issn} - {self.pid_v2 or self.pid_v3 or self.pid_generic}' - - @classmethod - def metadata(cls, collection=None): - qs = cls.objects.select_related('collection').only( - 'collection__acron3', - 'default_lang', - 'files', - 'pid_v2', - 'pid_v3', - 'pid_generic', - 'processing_date', - 'publication_date', - 'publication_year', - 'scielo_issn', - 'text_langs', - ) - - if collection: - qs = qs.filter(collection=collection) - - for a in qs.iterator(): - yield { - 'collection': a.collection.acron3, - 'default_lang': a.default_lang, - 'files': a.files, - 'pid_v2': a.pid_v2, - 'pid_v3': a.pid_v3, - 'pid_generic': a.pid_generic, - 'processing_date': a.processing_date, - 'publication_date': a.publication_date, - 'publication_year': a.publication_year, - 'scielo_issn': a.scielo_issn, - 'text_langs': a.text_langs, - } - - class Meta: - verbose_name = _('Article') - verbose_name_plural = _('Articles') - unique_together = ( - 'collection', - 'scielo_issn', - 'pid_v2', - 'pid_v3', - 'pid_generic', - ) - indexes = [ - models.Index(fields=['collection', 'scielo_issn'], name='collection_scielo_issn_idx'), - ] - diff --git a/article/tasks.py b/article/tasks.py deleted file mode 100644 index 3514fca..0000000 --- a/article/tasks.py +++ /dev/null @@ -1,259 +0,0 @@ -import logging - -from django.contrib.auth import get_user_model -from django.db.models import Q -from django.db import DataError -from django.utils.translation import gettext as _ - -from collection.models import Collection -from config import celery_app -from core.utils import date_utils -from core.utils.utils import _get_user - -from journal.models import Journal - -from tracker.models import ArticleEvent -from tracker.choices import ARTICLE_EVENT_TYPE_MULTIPLE_OBJS_RETURNED, ARTICLE_EVENT_TYPE_DATA_ERROR - -from . import models, utils - - -User = get_user_model() - -@celery_app.task(bind=True, name=_('Load article data from Article Meta'), timelimit=-1, queue='load') -def task_load_article_from_article_meta(self, from_date=None, until_date=None, days_to_go_back=None, collection=None, issn=None, force_update=True, user_id=None, username=None): - user = _get_user(self.request, username=username, user_id=user_id) - - from_date, until_date = date_utils.get_date_range_str(from_date, until_date, days_to_go_back) - logging.info(f'Loading articles from Article Meta. From: {from_date}, Until: {until_date}, Collection: {collection}, ISSN: {issn}.') - - offset = 0 - limit = 1000 - while True: - logging.info(f'{from_date}, {until_date}, {offset}, {limit}, {collection}, {issn}') - response = utils.fetch_article_meta_dict(from_date, until_date, offset=offset, limit=limit, collection=collection, issn=issn) - objects = response.get('objects') - if not objects: - break - - for obj in objects: - codes = obj.get('code_title') - - for issn_code in codes: - jou = Journal.objects.filter( - Q(issns__electronic_issn=issn_code) | - Q(issns__scielo_issn=issn_code) | - Q(issns__print_issn=issn_code) - ).first() - if not jou: - continue - - if not jou: - logging.info(f'Journal not found for ISSNs: {codes}') - continue - - col_obj = Collection.objects.get(acron3=obj.get('collection')) - if not col_obj: - logging.info(f'Collection not found: {obj.get("collection")}') - continue - - try: - article, created = models.Article.objects.get_or_create(collection=col_obj, scielo_issn=jou.scielo_issn, pid_v2=obj.get('code')) - if created or force_update: - article.files = obj.get('pdfs') or {} - article.processing_date = obj.get('processing_date') or '' - article.publication_date = obj.get('publication_date') or '' - article.publication_year = obj.get('publication_year') or '' - article.default_lang = obj.get('default_language') or '' - article.text_langs = obj.get('text_langs') or '' - - article.save() - logging.info(f'Article {"created" if created else "updated"}: {article}') - except models.Article.MultipleObjectsReturned as e: - logging.error(f'Error getting Article: {e}. Collection: {col_obj}, ISSN: {jou.scielo_issn}, PIDv2: {obj.get("code")}') - ArticleEvent.create( - event_type=ARTICLE_EVENT_TYPE_MULTIPLE_OBJS_RETURNED, - message=f'Error getting Article: {e}. Collection: {col_obj}, ISSN: {jou.scielo_issn}, PIDv2: {obj.get("code")}', - data=obj - ) - continue - except DataError as e: - logging.error(f'Error saving Article: {e}. Collection: {col_obj}, ISSN: {jou.scielo_issn}, PIDv2: {obj.get("code")}') - ArticleEvent.create( - event_type=ARTICLE_EVENT_TYPE_DATA_ERROR, - message=f'Error saving Article: {e}. Collection: {col_obj}, ISSN: {jou.scielo_issn}, PIDv2: {obj.get("code")}', - data=obj - ) - continue - - offset += limit - - return True - - -@celery_app.task(bind=True, name=_('Load article data from OPAC'), timelimit=-1, queue='load') -def task_load_article_from_opac(self, collection='scl', from_date=None, until_date=None, days_to_go_back=None, page=1, force_update=True, user_id=None, username=None): - user = _get_user(self.request, username=username, user_id=user_id) - - from_date, until_date = date_utils.get_date_range_str(from_date, until_date, days_to_go_back) - logging.info(f'Loading articles from OPAC. From: {from_date}, Until: {until_date}') - - while True: - response = utils.fetch_opac_dict(from_date, until_date, page=page) - - documents = response.get('documents') - - for doc_id, doc in documents.items(): - col_obj = Collection.objects.get(acron3=collection) - if not col_obj: - logging.error(f'Collection not found: {collection}') - continue - - journal = Journal.objects.get(collection=col_obj, acronym=doc.get('journal_acronym')) - if not journal: - logging.error(f'Journal not found: {doc.get("journal_acronym")}') - continue - - try: - article, created = models.Article.objects.get_or_create(collection=col_obj, scielo_issn=journal.scielo_issn, pid_v2=doc.get('pid_v2')) - - if created or force_update: - article.pid_v3 = doc.get('pid_v3') or '' - if not created: - article.pid_v2 = doc.get('pid_v2') or '' - article.publication_date = doc.get('publication_date') or article.publication_date or '' - article.default_lang = doc.get('default_language') or article.default_lang or '' - - try: - article.publication_year = article.publication_date[:4] - except IndexError: - article.publication_year = '' - - article.save() - logging.info(f'Article {"created" if created else "updated"}: {article}') - - except models.Article.MultipleObjectsReturned as e: - logging.error(f'Error getting Article: {e}. Collection: {col_obj}, Journal: {journal.scielo_issn}, PIDv2: {doc.get("pid_v2")}') - ArticleEvent.create( - event_type=ARTICLE_EVENT_TYPE_MULTIPLE_OBJS_RETURNED, - message=f'Error creating Article: {e}. Collection: {col_obj}, Journal: {journal.scielo_issn}, PIDv2: {doc.get("pid_v2")}', - data=doc - ) - continue - except DataError as e: - logging.error(f'Error saving Article: {e}. Collection: {col_obj}, Journal: {journal.scielo_issn}, PIDv2: {doc.get("pid_v2")}') - ArticleEvent.create( - event_type=ARTICLE_EVENT_TYPE_DATA_ERROR, - message=f'Error saving Article: {e}. Collection: {col_obj}, Journal: {journal.scielo_issn}, PIDv2: {doc.get("pid_v2")}', - data=doc - ) - continue - - page += 1 - if page > int(response.get('pages', 0)): - break - - return True - - -@celery_app.task(bind=True, name=_('Load preprint data from SciELO Preprints'), timelimit=-1, queue='load') -def task_load_preprints_from_preprints_api(self, from_date=None, until_date=None, days_to_go_back=None, force_update=True, user_id=None, username=None): - user = _get_user(self.request, username=username, user_id=user_id) - - from_date, until_date = date_utils.get_date_range_str(from_date, until_date, days_to_go_back) - logging.info(f'Loading preprints from SciELO Preprints. From: {from_date}, Until: {until_date}') - - col_obj = Collection.objects.get(acron3='preprints') - if not col_obj: - logging.error(f'Collection not found: preprints') - return False - - for record in utils.fetch_preprint_oai_pmh(from_date, until_date): - data = utils.extract_preprint_data(record) - - if not data.get('pid_generic'): - logging.error(f'Preprint ID not found in record: {record}') - continue - - try: - article, created = models.Article.objects.get_or_create(collection=col_obj, pid_generic=data['pid_generic']) - if created or force_update: - article.text_langs = data.get('text_langs') - article.default_lang = data.get('default_language') - article.publication_date = data.get('publication_date') - article.publication_year = data.get('publication_year') - - # Preprints do not have a scielo_issn yet - article.scielo_issn = '0000-0000' - - article.save() - logging.debug(f'Article {"created" if created else "updated"}: {article}') - except models.Article.MultipleObjectsReturned as e: - logging.error(f'Error creating Article: {e}. Collection: {col_obj}, PID: {data["pid_generic"]}') - ArticleEvent.create( - event_type=ARTICLE_EVENT_TYPE_MULTIPLE_OBJS_RETURNED, - message=f'Error creating Article: {e}. Collection: {col_obj}, PID: {data["pid_generic"]}', - data=data - ) - continue - except DataError as e: - logging.error(f'Error saving Article: {e}. Collection: {col_obj}, PID: {data["pid_generic"]}') - ArticleEvent.create( - event_type=ARTICLE_EVENT_TYPE_DATA_ERROR, - message=f'Error saving Article: {e}. Collection: {col_obj}, PID: {data["pid_generic"]}', - data=data - ) - continue - - -@celery_app.task(bind=True, name=_('Load dataset metadata from Dataverse'), timelimit=-1, queue='load') -def task_load_dataset_metadata_from_dataverse(self, from_date=None, until_date=None, days_to_go_back=None, force_update=True, user_id=None, username=None): - user = _get_user(self.request, username=username, user_id=user_id) - - from_date, until_date = date_utils.get_date_range_str(from_date, until_date, days_to_go_back) - logging.info(f'Loading dataset metadata from SciELO Data. From: {from_date}, Until: {until_date}') - - col_obj = Collection.objects.get(acron3='data') - if not col_obj: - logging.error(f'Collection not found: data') - return False - - for record in utils.fetch_dataverse_metadata(from_date, until_date): - dataset_doi = record.get('dataset_doi') - if not dataset_doi: - logging.error(f'Dataset DOI not found in record: {record}') - continue - - try: - dataset, created = models.Article.objects.get_or_create(collection=col_obj, pid_generic=dataset_doi) - if created or force_update: - dataset.publication_date = record.get('dataset_published') - - file_persistent_id = record.get('file_persistent_id') - file_id = record.get('file_id') - file_name = record.get('file_name') - file_url = record.get('file_url') - - if file_id: - dataset.files[file_id] = {'name': file_name, 'url': file_url, 'file_persisent_id': file_persistent_id} - - dataset.save() - logging.debug(f'Dataset {"created" if created else "updated"}: {dataset}') - except models.Article.MultipleObjectsReturned as e: - logging.error(f'Error creating Dataset: {e}. Collection: {col_obj}, PID: {dataset_doi}') - ArticleEvent.create( - event_type=ARTICLE_EVENT_TYPE_MULTIPLE_OBJS_RETURNED, - message=f'Error creating Dataset: {e}. Collection: {col_obj}, PID: {dataset_doi}', - data=record - ) - continue - except DataError as e: - logging.error(f'Error saving Dataset: {e}. Collection: {col_obj}, PID: {dataset_doi}') - ArticleEvent.create( - event_type=ARTICLE_EVENT_TYPE_DATA_ERROR, - message=f'Error saving Dataset: {e}. Collection: {col_obj}, PID: {dataset_doi}', - data=record - ) - continue - - return True diff --git a/article/tests.py b/article/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/article/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/article/utils.py b/article/utils.py deleted file mode 100644 index b9a094e..0000000 --- a/article/utils.py +++ /dev/null @@ -1,204 +0,0 @@ -import logging -import requests -import os - -from sickle import Sickle -from time import sleep - -from core.utils import standardizer - - -ARTICLEMETA_ENDPOINT = os.environ.get('ARTICLEMETA_COLLECT_URL', 'http://articlemeta.scielo.org/api/v1/article/counter_dict') -ARTICLEMETA_MAX_RETRIES = int(os.environ.get('ARTICLEMETA_MAX_RETRIES', 5)) -ARTICLEMETA_SLEEP_TIME = int(os.environ.get('ARTICLEMETA_SLEEP_TIME', 30)) - -OPAC_ENDPOINT = os.environ.get('OPAC_ENDPOINT', 'https://www.scielo.br/api/v1/counter_dict') -OPAC_MAX_RETRIES = int(os.environ.get('OPAC_MAX_RETRIES', 5)) -OPAC_SLEEP_TIME = int(os.environ.get('OPAC_SLEEP_TIME', 30)) - -OAI_PMH_PREPRINT_ENDPOINT = os.environ.get('OAI_PMH_PREPRINT_ENDPOINT', 'https://preprints.scielo.org/index.php/scielo/oai') -OAI_METADATA_PREFIX = os.environ.get('OAI_METADATA_PREFIX', 'oai_dc') -OAI_PMH_MAX_RETRIES = int(os.environ.get('OAI_PMH_MAX_RETRIES', 5)) - -DATAVERSE_ENDPOINT = os.environ.get('DATAVERSE_ENDPOINT', 'https://data.scielo.org/api') -DATAVERSE_ROOT_COLLECTION = os.environ.get('DATAVERSE_ROOT_COLLECTION', 'scielodata') -DATAVERSE_MAX_RETRIES = int(os.environ.get('DATAVERSE_MAX_RETRIES', 5)) -DATAVERSE_SLEEP_TIME = int(os.environ.get('DATAVERSE_SLEEP_TIME', 30)) - - -def fetch_article_meta_dict(from_date, until_date, offset=0, limit=1000, collection=None, issn=None): - for t in range(1, ARTICLEMETA_MAX_RETRIES + 1): - params = { - 'from': from_date, - 'until': until_date, - 'offset': offset, - 'limit': limit - } - - if collection: - params['collection'] = collection - - if issn: - params['issn'] = issn - - response = requests.get(ARTICLEMETA_ENDPOINT, params=params) - - try: - response.raise_for_status() - logging.info(response.url) - - except requests.exceptions.HTTPError: - logging.warning( - 'Failed to collect data from %s. Waiting %d seconds before retry %d of %d' % ( - response.url, - ARTICLEMETA_SLEEP_TIME, - t, - ARTICLEMETA_MAX_RETRIES - ) - ) - sleep(ARTICLEMETA_SLEEP_TIME) - - else: - return response.json() - - -def fetch_opac_dict(from_date, until_date, page=1): - for t in range(1, OPAC_MAX_RETRIES + 1): - params = { - 'begin_date': from_date, - 'end_date': until_date, - 'page': page - } - - response = requests.get(url=OPAC_ENDPOINT, params=params, verify=False) - - try: - response.raise_for_status() - logging.info(response.url) - - except requests.exceptions.HTTPError: - logging.warning('Não foi possível coletar dados de %s. Aguardando %d segundos para tentativa %d de %d' % (response.url, OPAC_SLEEP_TIME, t, OPAC_MAX_RETRIES)) - sleep(OPAC_SLEEP_TIME) - - else: - return response.json() - - -def fetch_preprint_oai_pmh(from_date, until_date): - oai_client = Sickle(endpoint=OAI_PMH_PREPRINT_ENDPOINT, max_retries=OAI_PMH_MAX_RETRIES, verify=False) - records = oai_client.ListRecords(**{ - 'metadataPrefix': OAI_METADATA_PREFIX, - 'from': from_date, - 'until': until_date, - }) - - for r in records: - yield r - - -def extract_preprint_data(record): - pid_generic = _extract_preprint_compatible_identifer(record.header.identifier) - text_langs = [standardizer.standardize_language_code(l) for l in record.metadata.get('language', [])] - publication_date = record.metadata.get('date', [''])[0] - default_language = text_langs[0] if text_langs else '' - publication_year = _extract_preprint_publication_year_from_date(publication_date) - - data = { - 'pid_generic': pid_generic, - 'text_langs': text_langs, - 'publication_date': publication_date, - 'default_language': default_language, - 'publication_year': publication_year - } - - return data - - -def _extract_preprint_compatible_identifer(pid_v2): - try: - # piv_v2 should be something like oai:ops.preprints.scielo.org:preprint/1195 - # we are using the last part of the string as the identifier - return pid_v2.split(':')[-1].split('/')[1] - except IndexError: - return '' - - -def _extract_preprint_publication_year_from_date(date_str): - try: - return date_str[:4] - except IndexError: - return '' - - -def fetch_dataverse_metadata(from_date=None, until_date=None): - def get_subdataverses(): - url = f"{DATAVERSE_ENDPOINT}/dataverses/{DATAVERSE_ROOT_COLLECTION}/contents" - try: - response = requests.get(url, timeout=DATAVERSE_SLEEP_TIME) - response.raise_for_status() - return response.json().get("data", []) - except requests.exceptions.RequestException as e: - logging.error(f"Error fetching subdataverses: {e}") - return [] - - def get_datasets(subdataverse_id): - url = f"{DATAVERSE_ENDPOINT}/dataverses/{subdataverse_id}/contents" - try: - response = requests.get(url, timeout=DATAVERSE_SLEEP_TIME) - response.raise_for_status() - return response.json().get("data", []) - except requests.exceptions.RequestException as e: - logging.error(f"Error fetching datasets for subdataverse {subdataverse_id}: {e}") - return [] - - def get_files(dataset_id): - url = f"{DATAVERSE_ENDPOINT}/datasets/{dataset_id}/versions/:latest/files" - try: - response = requests.get(url, timeout=DATAVERSE_SLEEP_TIME) - response.raise_for_status() - return response.json().get("data", []) - except requests.exceptions.RequestException as e: - logging.error(f"Error fetching files for dataset {dataset_id}: {e}") - return [] - - subdataverses = get_subdataverses() - - for subdataverse in subdataverses: - if subdataverse["type"] != "dataverse": - continue - - subdataverse_id = subdataverse["id"] - subdataverse_title = subdataverse["title"] - datasets = get_datasets(subdataverse_id) - - for dataset in datasets: - if dataset["type"] != "dataset": - continue - - dataset_id = dataset["id"] - doi = standardizer.standardize_doi(dataset.get("persistentUrl")) - if not doi: - logging.warning(f"Dataset {dataset_id} does not have a DOI.") - continue - - publication_date = dataset.get("publicationDate", None) - - if publication_date: - if (from_date and publication_date < from_date) or (until_date and publication_date > until_date): - continue - - files = get_files(dataset_id) - - for file in files: - file_persistent_id = file["dataFile"].get("persistentId", None) - file_persistent_id_stz = standardizer.standardize_pid_generic(file_persistent_id) if file_persistent_id else None - - yield { - "title": subdataverse_title, - "dataset_doi": doi, - "dataset_published": publication_date, - "file_id": file["dataFile"]["id"], - "file_name": file["label"], - "file_url": f"{DATAVERSE_ENDPOINT}/access/datafile/{file['dataFile']['id']}", - "file_persistent_id": file_persistent_id_stz, - } diff --git a/article/views.py b/article/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/article/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/article/wagtail_hooks.py b/article/wagtail_hooks.py deleted file mode 100644 index 4cf55bd..0000000 --- a/article/wagtail_hooks.py +++ /dev/null @@ -1,39 +0,0 @@ -from django.utils.translation import gettext_lazy as _ -from wagtail.snippets.views.snippets import SnippetViewSet -from wagtail.snippets.models import register_snippet - -from config.menu import get_menu_order - -from .models import Article - - -class ArticleSnippetViewSet(SnippetViewSet): - model = Article - icon = "folder-open-inverse" - menu_name = "article" - menu_label = _("Article") - menu_order = get_menu_order("article") - add_to_admin_menu = True - - list_display = ( - "collection", - "scielo_issn", - "pid_v2", - "pid_v3", - "pid_generic", - "files", - "publication_year", - ) - list_filter = ( - "collection", - "scielo_issn", - "publication_year", - ) - search_fields = ( - "scielo_issn", - "pid_v2", - "pid_v3", - "pid_generic", - ) - -register_snippet(ArticleSnippetViewSet) diff --git a/collection/tasks.py b/collection/tasks.py index 02fd0e7..19372de 100644 --- a/collection/tasks.py +++ b/collection/tasks.py @@ -1,14 +1,14 @@ from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ -from core.utils.utils import _get_user +from core.utils.request_utils import _get_user from collection.models import Collection from config import celery_app User = get_user_model() -@celery_app.task(bind=True, name=_('Load collection data')) +@celery_app.task(bind=True, name=_('[Collection] Load Collection Data')) def task_load_collections(self, user_id=None, username=None): user = _get_user(self.request, username=username, user_id=user_id) Collection.load(user) diff --git a/collection/wagtail_hooks.py b/collection/wagtail_hooks.py index e7b7e97..52b31a8 100644 --- a/collection/wagtail_hooks.py +++ b/collection/wagtail_hooks.py @@ -1,8 +1,5 @@ from django.utils.translation import gettext as _ from wagtail.snippets.views.snippets import SnippetViewSet -from wagtail.snippets.models import register_snippet - -from config.menu import get_menu_order from .models import Collection @@ -10,10 +7,8 @@ class CollectionSnippetViewSet(SnippetViewSet): model = Collection icon = "folder-open-inverse" - menu_name = 'collection' menu_label = _("Collection") - menu_order = get_menu_order("collection") - add_to_admin_menu = True + menu_order = 100 list_display = ( "main_name", @@ -57,6 +52,3 @@ class CollectionSnippetViewSet(SnippetViewSet): "updated_by", ) export_filename = "collections" - - -register_snippet(CollectionSnippetViewSet) diff --git a/config/settings/base.py b/config/settings/base.py index 4e96ed4..9638274 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -114,10 +114,8 @@ "core.users", "core_settings", # Your stuff: custom apps go here - "article", "collection", "core", - "journal", "log_manager", "log_manager_config", "metrics", diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 6957700..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = ./_build -APP = /app - -.PHONY: help livehtml apidocs Makefile - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -c . - -# Build, watch and serve docs with live reload -livehtml: - sphinx-autobuild -b html --host 0.0.0.0 --port 9000 --watch $(APP) -c . $(SOURCEDIR) $(BUILDDIR)/html - -# Outputs rst files from django application code -apidocs: - sphinx-apidoc -o $(SOURCEDIR)/api $(APP) - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -c . diff --git a/docs/__init__.py b/docs/__init__.py deleted file mode 100644 index 8772c82..0000000 --- a/docs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Included so that Django's startproject comment runs against the docs directory diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 51cd921..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,64 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. - -import os -import sys - -import django - -if os.getenv("READTHEDOCS", default=False) == "True": - sys.path.insert(0, os.path.abspath("..")) - os.environ["DJANGO_READ_DOT_ENV_FILE"] = "True" - os.environ["USE_DOCKER"] = "no" -else: - sys.path.insert(0, os.path.abspath("/app")) -os.environ["DATABASE_URL"] = "sqlite:///readthedocs.db" -os.environ["CELERY_BROKER_URL"] = os.getenv("REDIS_URL", "redis://redis:6379") -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") -django.setup() - -# -- Project information ----------------------------------------------------- - -project = "SciELO Core" -copyright = """2022, SciELO""" -author = "SciELO" - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", -] - -# Add any paths that contain templates here, relative to this directory. -# templates_path = ["_templates"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "alabaster" - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ["_static"] diff --git a/docs/howto.rst b/docs/howto.rst deleted file mode 100644 index 9fae300..0000000 --- a/docs/howto.rst +++ /dev/null @@ -1,38 +0,0 @@ -How To - Project Documentation -====================================================================== - -Get Started ----------------------------------------------------------------------- - -Documentation can be written as rst files in `core/docs`. - - -To build and serve docs, use the commands:: - - docker compose -f local.yml up docs - - - -Changes to files in `docs/_source` will be picked up and reloaded automatically. - -`Sphinx `_ is the tool used to build documentation. - -Docstrings to Documentation ----------------------------------------------------------------------- - -The sphinx extension `apidoc `_ is used to automatically document code using signatures and docstrings. - -Numpy or Google style docstrings will be picked up from project files and availble for documentation. See the `Napoleon `_ extension for details. - -For an in-use example, see the `page source <_sources/users.rst.txt>`_ for :ref:`users`. - -To compile all docstrings automatically into documentation source files, use the command: - :: - - make apidocs - - -This can be done in the docker container: - :: - - docker run --rm docs make apidocs diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index b6c6ded..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. SciELO Content Manager documentation master file, created by - sphinx-quickstart. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to SciELO Core's documentation! -====================================================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - howto - users - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 4f70eed..0000000 --- a/docs/make.bat +++ /dev/null @@ -1,46 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -c . -) -set SOURCEDIR=_source -set BUILDDIR=_build -set APP=..\core - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.Install sphinx-autobuild for live serving. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -b %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:livehtml -sphinx-autobuild -b html --open-browser -p 9000 --watch %APP% -c . %SOURCEDIR% %BUILDDIR%/html -GOTO :EOF - -:apidocs -sphinx-apidoc -o %SOURCEDIR%/api %APP% -GOTO :EOF - -:help -%SPHINXBUILD% -b help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/users.rst b/docs/users.rst deleted file mode 100644 index 21e08aa..0000000 --- a/docs/users.rst +++ /dev/null @@ -1,15 +0,0 @@ - .. _users: - -Users -====================================================================== - -Starting a new project, it’s highly recommended to set up a custom user model, -even if the default User model is sufficient for you. - -This model behaves identically to the default user model, -but you’ll be able to customize it in the future if the need arises. - -.. automodule:: core.users.models - :members: - :noindex: - diff --git a/journal/__init__.py b/journal/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/journal/admin.py b/journal/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/journal/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/journal/apps.py b/journal/apps.py deleted file mode 100644 index e10a171..0000000 --- a/journal/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class JournalConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - name = "journal" diff --git a/journal/migrations/0001_initial.py b/journal/migrations/0001_initial.py deleted file mode 100644 index 7164bbc..0000000 --- a/journal/migrations/0001_initial.py +++ /dev/null @@ -1,122 +0,0 @@ -# Generated by Django 5.0.7 on 2025-02-07 17:50 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - ("collection", "0001_initial"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name="Journal", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "created", - models.DateTimeField( - auto_now_add=True, verbose_name="Creation date" - ), - ), - ( - "updated", - models.DateTimeField( - auto_now=True, verbose_name="Last update date" - ), - ), - ( - "scielo_issn", - models.CharField(max_length=9, verbose_name="SciELO ISSN"), - ), - ( - "issns", - models.JSONField( - blank=True, default=dict, null=True, verbose_name="ISSNs" - ), - ), - ( - "acronym", - models.CharField( - blank=True, - default="", - max_length=32, - null=True, - verbose_name="Journal Acronym", - ), - ), - ( - "title", - models.CharField(max_length=255, verbose_name="Journal Title"), - ), - ( - "publisher_name", - models.JSONField( - blank=True, - default=list, - null=True, - verbose_name="Publisher Name", - ), - ), - ( - "subject_areas", - models.JSONField( - default=list, verbose_name="Subject Areas (CAPES)" - ), - ), - ( - "wos_subject_areas", - models.JSONField(default=list, verbose_name="Subject Areas (WoS)"), - ), - ( - "collection", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="collection.collection", - verbose_name="Collection", - ), - ), - ( - "creator", - models.ForeignKey( - editable=False, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="%(class)s_creator", - to=settings.AUTH_USER_MODEL, - verbose_name="Creator", - ), - ), - ( - "updated_by", - models.ForeignKey( - blank=True, - editable=False, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="%(class)s_last_mod_user", - to=settings.AUTH_USER_MODEL, - verbose_name="Updater", - ), - ), - ], - options={ - "verbose_name": "Journal", - "verbose_name_plural": "Journals", - "unique_together": {("collection", "scielo_issn", "acronym")}, - }, - ), - ] diff --git a/journal/migrations/0002_alter_journal_scielo_issn.py b/journal/migrations/0002_alter_journal_scielo_issn.py deleted file mode 100644 index 07cf94f..0000000 --- a/journal/migrations/0002_alter_journal_scielo_issn.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 5.0.7 on 2025-06-12 17:16 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("journal", "0001_initial"), - ] - - operations = [ - migrations.AlterField( - model_name="journal", - name="scielo_issn", - field=models.CharField( - db_index=True, max_length=9, verbose_name="SciELO ISSN" - ), - ), - ] diff --git a/journal/migrations/__init__.py b/journal/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/journal/models.py b/journal/models.py deleted file mode 100644 index 0d830e9..0000000 --- a/journal/models.py +++ /dev/null @@ -1,100 +0,0 @@ -from django.db import models -from django.utils.translation import gettext_lazy as _ - -from core.models import CommonControlField -from collection.models import Collection - - -class Journal(CommonControlField): - collection = models.ForeignKey( - Collection, - verbose_name=_('Collection'), - on_delete=models.CASCADE, - blank=False, - null=False, - db_index=True, - ) - - scielo_issn = models.CharField( - verbose_name=_('SciELO ISSN'), - max_length=9, - blank=False, - null=False, - db_index=True, - ) - - issns = models.JSONField( - verbose_name=_('ISSNs'), - null=True, - blank=True, - default=dict, - ) - - acronym = models.CharField( - verbose_name=_('Journal Acronym'), - max_length=32, - blank=True, - null=True, - default='', - ) - - title = models.CharField( - verbose_name=_('Journal Title'), - max_length=255, - blank=False, - null=False, - ) - - publisher_name = models.JSONField( - verbose_name=_('Publisher Name'), - blank=True, - null=True, - default=list, - ) - - subject_areas = models.JSONField( - verbose_name=_('Subject Areas (CAPES)'), - null=False, - blank=False, - default=list, - ) - - wos_subject_areas = models.JSONField( - verbose_name=_('Subject Areas (WoS)'), - null=False, - blank=False, - default=list, - ) - - def __str__(self): - return f'{self.collection.acron2} - {self.scielo_issn} - {self.acronym}' - - @classmethod - def metadata(cls, collection=None): - queryset = cls.objects.all() - if collection: - queryset = queryset.filter(collection=collection) - - for journal in queryset.only( - 'acronym', 'collection__acron3', 'issns', 'publisher_name', - 'scielo_issn', 'subject_areas', 'title', 'wos_subject_areas' - ): - yield { - 'acronym': journal.acronym, - 'collection': journal.collection.acron3, - 'issns': set([v for v in journal.issns.values() if v]), - 'publisher_name': journal.publisher_name, - 'scielo_issn': journal.scielo_issn, - 'subject_areas': journal.subject_areas, - 'title': journal.title, - 'wos_subject_areas': journal.wos_subject_areas, - } - - class Meta: - verbose_name = _('Journal') - verbose_name_plural = _('Journals') - unique_together = ( - 'collection', - 'scielo_issn', - 'acronym', - ) diff --git a/journal/tasks.py b/journal/tasks.py deleted file mode 100644 index 71681cb..0000000 --- a/journal/tasks.py +++ /dev/null @@ -1,56 +0,0 @@ -import logging - -from django.contrib.auth import get_user_model -from django.db import IntegrityError -from django.utils import timezone -from django.utils.translation import gettext as _ - -from collection.models import Collection -from config import celery_app -from core.utils.utils import _get_user - -from . import models, utils - - -User = get_user_model() - - -@celery_app.task(bind=True, name=_('Load journal data from Article Meta'), queue='load') -def task_load_journal_data_from_article_meta(self, collections=[], force_update=True, user_id=None, username=None, mode='thrift'): - user = _get_user(user_id, username) - - for col in collections or Collection.acron3_list(): - for j in utils.fetch_article_meta_journals(collection=col, mode=mode): - collection = Collection.objects.get(acron3=j.collection_acronym) - if not collection: - logging.error(f'Collection {j.collection_acronym} does not exist') - continue - - try: - journal, created = models.Journal.objects.get_or_create(collection=collection, scielo_issn=j.scielo_issn) - except IntegrityError as e: - logging.error(f'Journal {j} has not been created due to error: {e}') - continue - - if created: - journal.creator = user - journal.created = timezone.now() - - if created or force_update: - journal.updated_by = user - journal.updated = timezone.now() - journal.issns = { - 'electronic_issn': j.electronic_issn or '', - 'print_issn': j.print_issn or '', - 'scielo_issn': j.scielo_issn - } - journal.acronym = j.acronym - journal.title = j.title - journal.publisher_name = j.publisher_name or '' - journal.subject_areas = j.subject_areas or [] - journal.wos_subject_areas = j.wos_subject_areas or [] - logging.info(f'Journal {"created" if created else "updated"}: {journal}') - - journal.save() - - return True diff --git a/journal/tests.py b/journal/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/journal/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/journal/utils.py b/journal/utils.py deleted file mode 100644 index 8a80521..0000000 --- a/journal/utils.py +++ /dev/null @@ -1,19 +0,0 @@ -from articlemeta.client import ThriftClient, RestfulClient - - -def fetch_article_meta_journals(collection='scl', mode='rest'): - """ - Fetches article metadata from journals. - - Returns - ------- - list - A list of article metadata. - """ - if mode == 'rest': - am = RestfulClient() - elif mode == 'thrift': - am = ThriftClient() - - for j in am.journals(collection=collection): - yield j diff --git a/journal/views.py b/journal/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/journal/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/journal/wagtail_hooks.py b/journal/wagtail_hooks.py deleted file mode 100644 index 725b370..0000000 --- a/journal/wagtail_hooks.py +++ /dev/null @@ -1,40 +0,0 @@ -from django.utils.translation import gettext_lazy as _ -from wagtail.snippets.views.snippets import SnippetViewSet -from wagtail.snippets.models import register_snippet - -from config.menu import get_menu_order - -from .models import Journal - - -class JournalSnippetViewSet(SnippetViewSet): - model = Journal - icon = "folder-open-inverse" - menu_name = "journal" - menu_label = _("Journal") - menu_order = get_menu_order('journal') - add_to_admin_menu = True - - list_display = ( - "collection", - "scielo_issn", - "acronym", - "title", - "issns", - "publisher_name", - "subject_areas", - "wos_subject_areas", - ) - list_filter = ( - "collection", - ) - search_fields = ( - "issns", - "acronym", - "publisher_name", - "subject_areas", - "wos_subject_areas", - ) - - -register_snippet(JournalSnippetViewSet) diff --git a/merge_production_dotenvs_in_dotenv.py b/merge_production_dotenvs_in_dotenv.py deleted file mode 100644 index d1170ef..0000000 --- a/merge_production_dotenvs_in_dotenv.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -from pathlib import Path -from typing import Sequence - -import pytest - -ROOT_DIR_PATH = Path(__file__).parent.resolve() -PRODUCTION_DOTENVS_DIR_PATH = ROOT_DIR_PATH / ".envs" / ".production" -PRODUCTION_DOTENV_FILE_PATHS = [ - PRODUCTION_DOTENVS_DIR_PATH / ".django", - PRODUCTION_DOTENVS_DIR_PATH / ".postgres", -] -DOTENV_FILE_PATH = ROOT_DIR_PATH / ".env" - - -def merge( - output_file_path: str, merged_file_paths: Sequence[str], append_linesep: bool = True -) -> None: - with open(output_file_path, "w") as output_file: - for merged_file_path in merged_file_paths: - with open(merged_file_path, "r") as merged_file: - merged_file_content = merged_file.read() - output_file.write(merged_file_content) - if append_linesep: - output_file.write(os.linesep) - - -def main(): - merge(DOTENV_FILE_PATH, PRODUCTION_DOTENV_FILE_PATHS) - - -@pytest.mark.parametrize("merged_file_count", range(3)) -@pytest.mark.parametrize("append_linesep", [True, False]) -def test_merge(tmpdir_factory, merged_file_count: int, append_linesep: bool): - tmp_dir_path = Path(str(tmpdir_factory.getbasetemp())) - - output_file_path = tmp_dir_path / ".env" - - expected_output_file_content = "" - merged_file_paths = [] - for i in range(merged_file_count): - merged_file_ord = i + 1 - - merged_filename = ".service{}".format(merged_file_ord) - merged_file_path = tmp_dir_path / merged_filename - - merged_file_content = merged_filename * merged_file_ord - - with open(merged_file_path, "w+") as file: - file.write(merged_file_content) - - expected_output_file_content += merged_file_content - if append_linesep: - expected_output_file_content += os.linesep - - merged_file_paths.append(merged_file_path) - - merge(output_file_path, merged_file_paths, append_linesep) - - with open(output_file_path, "r") as output_file: - actual_output_file_content = output_file.read() - - assert actual_output_file_content == expected_output_file_content - - -if __name__ == "__main__": - main() diff --git a/start-dev.sh b/start-dev.sh deleted file mode 100644 index 92d064a..0000000 --- a/start-dev.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Change this value to the local ethernet. -ethernet=wlp0s20f3 - -# Linux IP. -export IP=$(/sbin/ip -o -4 addr list $ethernet | awk '{print $4}' | cut -d/ -f1) - -# Mac OS IP. -#export IP=$(ifconfig $ethernet | grep inet | grep -v inet6 | awk '{print $2}') - -export DATABASE_URL=postgres://GVRFlLmcCNfGLhsFvSnCioYOPJPYpyfj:BQ4hSUL4rdj5WZLdR8ilDLRQMvCtzo0caMaXDO0olGsmycQjlcZlTVK9DepZR8kk@$IP:5432/scielo_core -export CELERY_BROKER_URL=redis://$IP:6379/0 -export USE_DOCKER=no -export IPYTHONDIR=/app/.ipython -export REDIS_URL=redis://$IP:6379/0 -export CELERY_FLOWER_USER=PhFRdLexbrsBvrrbSXxjcMMOcVOavCrZ -export CELERY_FLOWER_PASSWORD=QgScyefPrYhHgO6onW61u0nazc5xdBuP4sM7jMRrBBFuA2RjsFhZLp7xbVYZbrwR -export EMAIL_HOST=$IP -export SOLR_URL=http://$IP:8983/solr/ - - -docker stop scielo_core_local_django -# workon scms -python manage.py runserver_plus 0.0.0.0:8000