diff options
| author | Tom Cooks <tommasogagliardi+github@gmail.com> | 2026-10-10 15:42:04 -0400 |
|---|---|---|
| committer | Tom Cooks <tommasogagliardi+github@gmail.com> | 2026-10-10 15:42:04 -0400 |
| commit | 901bf58c5cf9151942f8c4cd4e1427a02dec4b52 (patch) | |
| tree | e9e312fb9893881f142a6f79e30417cd0c3f2be8 | |
| parent | 6db1714d0354d1e1105e330ce84c46a54487c3bb (diff) | |
| download | reccoon-901bf58c5cf9151942f8c4cd4e1427a02dec4b52.tar.gz | |
RECCoon 1.0.0: rebrand to Tom Cooks, Parakeet-only, GPLv3, F-Droid-ready
51 files changed, 3704 insertions, 446 deletions
@@ -9,7 +9,6 @@ .externalNativeBuild .cxx -# Large prebuilt binaries — see TODO.md ("Build / distribution"). -# The sherpa-onnx AAR (~50 MB) and built APKs are not tracked in git yet. -/app/libs/*.aar +# Build outputs. The sherpa-onnx runtime is fetched from JitPack, so no +# prebuilt binary is committed here. /dist/ @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<https://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<https://www.gnu.org/licenses/why-not-lgpl.html>. @@ -4,22 +4,34 @@ A minimal Android app that records uncompressed WAV audio and transcribes it on device. - **Format:** 44.1 kHz, 16-bit PCM (stereo when the device supports it), no compression +- **Compact mode:** an optional switch saves a small AAC-LC `.m4a` instead of the WAV + (~22 MB/hour mono instead of ~317 MB); transcription still works because the + decoder streams the M4A back to PCM (`PcmAudioSource`) - **Controls:** one big round button — tap to start recording (timer starts), tap again to stop and save - **Meters:** live scrolling waveform plus left/right peak level meters with a clip indicator while recording - **Mono switch:** optionally record a single channel (half the file size) +- **Swap L/R:** exchange the stereo channels when a device reports them inverted in + landscape; applies to the meters and the saved file +- **Mic picker:** choose Auto, the built-in mic, or a specific input (Bluetooth SCO, + USB, wired headset ...) — the BT-Mic-Force feature folded into RECCoon +- **Live monitoring:** an optional **Monitor** switch plays the microphone through a + chosen output (built-in speaker, wired, USB, Bluetooth ...). Use headphones — the + speaker causes feedback. A dedicated thread and a bounded queue keep a slow output + from ever stalling the recording +- **Discard:** stop a recording and throw it away without saving - **Markers:** place labelled cue points while recording; they are embedded in the WAV (`cue ` + `LIST/adtl`) and shown in the player - **Silence:** optional Do Not Disturb total silence (no notifications, no vibration) while recording - **Recordings list:** scrollable list at the bottom of the main screen, newest first - **Player:** scrubber, ±10 s jumps, speed control, delete / rename / share / export, no autoplay - **Transcription:** on-device, offline speech recognition with [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) using multilingual - Whisper — works with **English and Italian** -- **Name:** RECCoon · **by** wuhei · **version** 1.4.0-alpha + **NVIDIA Parakeet v3** — works with English, Italian and 23 other languages +- **Name:** RECCoon · **by** Tom Cooks · **version** 1.0.0 - **Icon:** raccoon with a microphone ## Requirements -- Android 8.0 (API 26) or newer, arm64-v8a or x86_64 +- Android 8.0 (API 26) or newer, arm64-v8a - `RECORD_AUDIO` permission - On Android 9 and older, `WRITE_EXTERNAL_STORAGE` is also requested - Internet access the first time a transcription model is downloaded @@ -37,16 +49,19 @@ Outputs: - `app/build/outputs/apk/debug/app-debug.apk` - `app/build/outputs/apk/release/app-release.apk` -A prebuilt copy is available at `dist/RECCoon-1.4.0-alpha.apk`. Note that the -sherpa-onnx AAR in `app/libs/` and the APK in `dist/` are **not** tracked in -git because they are large; see `TODO.md`. +A prebuilt copy is available at `dist/RECCoon-1.0.0.apk` (release, signed with +the debug key). The APK in `dist/` is **not** tracked in git. The sherpa-onnx +runtime comes from [JitPack](https://jitpack.io) (an F-Droid-trusted Maven +repository), so no prebuilt binary lives in this repository. ## Recording -- `WavRecorder` uses `AudioRecord` (MIC source) to capture 16-bit PCM at - 44100 Hz into a temporary file, then rewrites a correct 44-byte RIFF/WAVE - header when recording stops. It uses stereo when available and falls back to - mono, and reports per-channel peak levels to the UI. +- `WavRecorder` uses `AudioRecord` (MIC source, or `VOICE_COMMUNICATION` for + Bluetooth SCO) to capture 16-bit PCM at 44100 Hz into a temporary file, then + rewrites a correct 44-byte RIFF/WAVE header when recording stops. It uses + stereo when available and falls back to mono, and reports per-channel peak + levels to the UI. A selected input device is applied with + `AudioRecord.setPreferredDevice`. - `RecordingService` is a microphone-typed foreground service started while recording, so capture continues when the app is in the background. - `MainActivity` handles the runtime permissions, the timer, the level meters @@ -79,32 +94,65 @@ git because they are large; see `TODO.md`. (MediaStore or file), **Export** (writes `.txt` and `.srt` to Downloads) and **Delete** (removes the WAV and its sidecars). - Transcript words are clickable: tapping a word seeks the player to it. Word - timings come from Whisper token timestamps when available, otherwise the + timings come from the model's token timestamps when available, otherwise the segment is divided between its words. ## On-device transcription -- `TranscriptionEngine` streams the WAV, resamples it to 16 kHz and segments it - with Silero VAD, then decodes each speech segment with a multilingual Whisper - model through sherpa-onnx. Streaming keeps memory bounded for long - recordings (offline Whisper only handles 30 s at a time). -- `ModelRepository` downloads the models on first use into app-private storage: +- `TranscriptionEngine` reads the recording through `PcmAudioSource`, which + parses WAV directly and decodes compressed files (AAC/M4A) with + `MediaExtractor` + `MediaCodec`, resamples to 16 kHz and segments the audio + with Silero VAD, then decodes each speech segment with the Parakeet v3 + transducer through sherpa-onnx. Streaming keeps memory bounded for long + recordings. +- `ModelRepository` downloads the model on first use: | Model | Size | Notes | |---|---|---| - | Whisper tiny (int8, multilingual) | ~99 MB | faster, lower accuracy | - | Whisper base (int8, multilingual) | ~154 MB | default, better Italian | - - Both include the small Silero VAD model (~0.6 MB). Files come from the - sherpa-onnx Whisper model repos on Hugging Face and the sherpa-onnx GitHub - release. -- Language can be forced to English (`en`) or Italian (`it`), or left on - **Auto detect**. + | Parakeet v3 (int8, 25 languages) | ~670 MB | only model, multilingual, large download | + + It includes the small Silero VAD model (~0.6 MB). Files come from the + sherpa-onnx model repo on Hugging Face and the sherpa-onnx GitHub release. + Models are stored in the app-specific external folder + `/sdcard/Android/data/com.tomcooks.reccoon/files/models/` so they can be + side-loaded with `adb push` or copied over USB instead of downloaded in-app. +- Parakeet v3 is a transducer, so it is loaded with `model_type="nemo_transducer"` + and handles language detection itself (no language picker). It needs ~670 MB of + storage and a large-heap device. - Transcripts are stored as JSON sidecars in app-private storage (`filesDir/transcripts/<recording>.json`). +## Compact (compressed) recordings + +- The **Compressed (small M4A)** checkbox transcodes the finished WAV to + AAC-LC (48 kbps per channel) with `AudioCompressor` and saves that instead. + Markers are kept as a JSON sidecar. Use it when storage matters more than + bit-exact audio. +- `tools/whistle-eval/` contains a desktop harness that compares Whisper, + Parakeet and Cactus Whistle; see its README for the results. + +## Live monitoring and discard + +- **Monitor** plays the incoming audio back through the selected output device while + recording. The output picker mirrors the mic picker (Auto, phone speaker, wired, + USB, Bluetooth). `LiveMonitor` owns an `AudioTrack` on its own thread with a small + bounded queue, so a slow output (for example Bluetooth A2DP) drops monitor chunks + instead of blocking the capture loop and corrupting the recording. Headphones are + strongly recommended — the phone speaker feeds back. +- **Discard** stops the recording and deletes the temporary file without saving it, + for takes that are obviously bad. + +## License + +RECCoon is free software, released under the **GNU General Public License, +version 3 or later** (see `LICENSE`). The Parakeet v3 model is +[NVIDIA Parakeet TDT 0.6B v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3), +licensed CC-BY-4.0. + ## Native libraries and APK size -The APK bundles the sherpa-onnx/onnxruntime native libraries for `arm64-v8a` -and `x86_64`, so it is around 72 MB. Removing `x86_64` from the `abiFilters` -in `app/build.gradle.kts` roughly halves the APK size for phone-only builds. +The APK bundles the sherpa-onnx/onnxruntime native libraries only for +`arm64-v8a` (`abiFilters` in `app/build.gradle.kts`), which keeps it around +35–40 MB. Including `x86_64` as well would roughly double it; that is only +needed for x86 emulators. The ~670 MB Parakeet model is downloaded at runtime, +not bundled. @@ -1,5 +1,118 @@ # RECCoon TODO +## Requested (2026-10-05) + +### 1. Compare transcription models (Whisper / Parakeet / Whistle) + +#### 1a. Parakeet TDT 0.6B v3 — **done, integrated** + +- [x] **Test NVIDIA Parakeet v3** (`csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8`). + - [x] Added `parakeet_baseline.py` to `tools/whistle-eval/` and wired it into + `evaluate.py`. + - [x] Result (same six clips): average WER **3.7%** vs Whisper base 11.4% and + tiny 13.9%. Italian **2.8%** (vs base 16.7%) and **0%** on the noisy + Italian clip. The only errors are number formatting. + - [x] sherpa-onnx already supports it (`model_type="nemo_transducer"`), so no new + runtime is needed. + - [x] Integrated into the app: `ModelRepository.Model.PARAKEET_V3`, + `TranscriptionEngine` builds an `OfflineTransducerModelConfig`, and the + player model spinner lists "Parakeet v3 (best)". + - [ ] Verify decoding and memory use on a real phone; the int8 model is + ~670 MB (~652 MB encoder) and needs a large-heap device. + - [x] Removed the Whisper tiny/base options: Parakeet v3 is now the only model + (and therefore the default). The language picker was removed too, since + Parakeet handles language by itself. The `Model` enum keeps room for more + models later. + - [ ] Consider a smaller multilingual Parakeet/Canary variant if device RAM is tight. + +#### 1b. Cactus Whistle — tested, not integrated + +- [~] **Check whether Cactus Whistle transcribes better than multilingual Whisper.** + Reference: <https://huggingface.co/mrfakename/whistle-ONNX> (ONNX export of + `Cactus-Compute/whistle`; supports en/de/fr/es/it/nl/pl). Motivation: Whisper + makes many mistakes, and a smaller model could replace the ~50 MB sherpa-onnx + AAR and fix the git push problem. + - [x] Build a desktop evaluation harness under `tools/whistle-eval/` (`setup.sh`, + `evaluate.py`, `whisper_baseline.py`, `build-data.mjs`). + - [x] Baseline + metric: the harness runs Whisper tiny/base through the same + sherpa-onnx 1.13.8 version the app bundles and reports WER. + - [x] First result (2026-10-05, six TTS clips, EN + IT): Whistle is **competitive + but not a clear win** — average WER 11.8% vs Whisper base 11.4% and tiny + 13.9%. On Italian Whistle was best (16.1% vs base 16.7%, tiny 23.2%); on + English tiny was best. See `tools/whistle-eval/README.md`. + - [ ] Decide the Android port after running the harness on **real RECCoon + recordings** — the TTS result is too close to call. + - [x] Feasibility: Whistle is a custom *Needle* encoder/decoder, **not** Whisper, so + sherpa-onnx cannot load it. An Android port needs `onnxruntime-android` plus + the mel filterbank, BPE tokenizer and engram features from `js/` / + `whistle.pack`. + - [x] Distribution: `whistle.pack` is ~17 MB (fits GitHub); the fp32 ONNX data is + ~145 MB. `build-data.mjs` reconstructs the `.onnx.data` files from the pack + byte-exactly, so only the pack needs to ship. + - [x] Decision: **do not add Whistle to the model picker** — Parakeet v3 is both + more accurate and supported by sherpa-onnx, so the custom-architecture port + is not worth it. + +### 2. Compact (compressed) recordings + +- [~] **Switch on the main screen to save very small compressed files instead of WAV.** + - [x] Codec: AAC-LC in MP4 (`.m4a`) for universal `MediaCodec` support; bitrate is + 48 kbps per channel (`AudioCompressor`). + - [x] Added a **Compressed (small M4A)** checkbox under the record button, persisted + as `compress_recordings`. + - [x] The temporary WAV is transcoded to `.m4a` after stopping and then deleted; the + M4A is saved to Downloads instead of the WAV. + - [x] Markers still work through the existing JSON sidecar (M4A has no `cue ` chunk). + - [x] `TranscriptionEngine` now streams through `PcmAudioSource`, which decodes + compressed files with `MediaExtractor`/`MediaCodec`, so transcription still + works on M4A. + - [ ] Measure the size win and the transcription-quality impact on a real device and + write the numbers here. Expected size: ~22 MB/hour mono instead of ~317 MB. + +### 3. Forced microphone input (BT-Mic-Force) + +- [~] **Integrate the BT-Mic-Force mic-forcing feature into RECCoon.** + - [x] Added a **Mic** spinner (Auto / Phone mic / detected inputs) to the main + screen. + - [x] The choice is routed into `WavRecorder` via `AudioRecord.setPreferredDevice` + and the `AudioManager` SCO mode (`setMode`, `startBluetoothSco`, + `setBluetoothScoOn`). + - [x] Added `MODIFY_AUDIO_SETTINGS`, `BLUETOOTH_CONNECT` and (pre-31) + `BLUETOOTH`/`BLUETOOTH_ADMIN` permissions; `BLUETOOTH_CONNECT` is requested + only when a Bluetooth input is selected. + - [x] Reused the existing `AudioRecord` capture path; no second foreground service. + - [ ] Verify that Bluetooth SCO actually routes audio on a real device. + +### 4. Live monitoring, output picker, and quick discard + +- [x] **Live monitoring** (`Monitor` checkbox + output picker). `LiveMonitor` plays the + captured PCM through `AudioTrack` on a dedicated thread with a bounded queue, so + a slow output never stalls the recording. Output devices are listed like the mic + picker (Auto / phone speaker / wired / USB / Bluetooth). Feedback warning when + monitoring is enabled. +- [x] **Stop and discard.** A **Discard** button stops the recording and deletes the + temporary file without saving. +- [ ] Verify latency and routing on real devices (wired, Bluetooth A2DP, BT LE). + Bluetooth A2DP adds unavoidable latency; note whether it is usable. +- [ ] Consider requesting audio focus and muting monitoring automatically when the + selected output is the built-in speaker. + +### 5. Rotation feedback (Alberto, from 1.4.0-alpha) + +- [x] **Level meters look upside down when the phone is turned.** Cause: both + activities were locked to `screenOrientation="portrait"`, so nothing followed + the device. + - [x] Removed the portrait lock so the UI rotates with the device. + - [x] Added `configChanges="orientation|screenSize|..."` so a rotation does not + recreate the activity and interrupt an in-progress recording. + - [x] **L/R swapped in landscape.** Android exposes the two mic channels in a fixed + device order; it does not tell the app which physical mic is which, so this is + not reliably auto-detectable. Added a persisted **Swap L/R** checkbox that + exchanges the channels in `WavRecorder`, so both the meters and the saved WAV + match the user's left/right. + - [ ] Confirm on a real device that rotating mid-recording keeps the recording and + the meters upright, and that Swap L/R fixes the inverted meters. + ## Build / distribution - [ ] **The sherpa-onnx binaries are too large to keep in git.** The Android diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 1e4768a..1f5fb20 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -3,7 +3,7 @@ plugins { } android { - namespace = "com.wuhei.reccoon" + namespace = "com.tomcooks.reccoon" compileSdk { version = release(37) { @@ -12,15 +12,16 @@ android { } defaultConfig { - applicationId = "com.wuhei.reccoon" + applicationId = "com.tomcooks.reccoon" minSdk = 26 targetSdk = 35 - versionCode = 6 - versionName = "1.4.0-alpha" + versionCode = 1 + versionName = "1.0.0" ndk { - // Prebuilt sherpa-onnx / onnxruntime native libraries. - abiFilters += listOf("arm64-v8a", "x86_64") + // Prebuilt sherpa-onnx / onnxruntime native libraries. Only arm64 is + // shipped; x86_64 was only useful for emulators and doubled the APK. + abiFilters += listOf("arm64-v8a") } } @@ -58,8 +59,10 @@ dependencies { // Kotlin runtime required by the sherpa-onnx Kotlin API. implementation("org.jetbrains.kotlin:kotlin-stdlib:2.4.20") - // On-device speech recognition (Whisper multilingual: English + Italian). - implementation(files("libs/sherpa-onnx-1.13.8.aar")) + // On-device speech recognition runtime (loads the Parakeet ONNX graphs). + // Pulled from JitPack (an F-Droid-trusted Maven repository) so no prebuilt + // binary has to live in this repo. Byte-identical to the official AAR. + implementation("com.github.k2-fsa.sherpa-onnx:sherpa-onnx:1.13.8@aar") testImplementation("junit:junit:4.13.2") } diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 791d668..eecbcc1 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -2,11 +2,22 @@ <manifest xmlns:android="http://schemas.android.com/apk/res/android"> <uses-permission android:name="android.permission.RECORD_AUDIO" /> + <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" /> + <uses-permission + android:name="android.permission.BLUETOOTH_CONNECT" /> + <uses-permission + android:name="android.permission.BLUETOOTH" + android:maxSdkVersion="30" /> + <uses-permission + android:name="android.permission.BLUETOOTH_ADMIN" + android:maxSdkVersion="30" /> <uses-permission android:name="android.permission.INTERNET" /> <uses-permission android:name="android.permission.ACCESS_NOTIFICATION_POLICY" /> <uses-permission android:name="android.permission.FOREGROUND_SERVICE" /> <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" /> + <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" /> <uses-permission android:name="android.permission.POST_NOTIFICATIONS" /> + <uses-permission android:name="android.permission.WAKE_LOCK" /> <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" android:maxSdkVersion="28" /> @@ -24,10 +35,16 @@ android:supportsRtl="true" android:theme="@style/Theme.Reccoon"> + <!-- + No screenOrientation lock: the UI rotates with the device so the level + meters and waveform stay upright. configChanges keeps the activity (and + the in-progress WavRecorder) alive across a rotation instead of recreating + it. + --> <activity android:name=".MainActivity" - android:exported="true" - android:screenOrientation="portrait"> + android:configChanges="orientation|screenSize|screenLayout|smallestScreenSize|keyboardHidden|uiMode" + android:exported="true"> <intent-filter> <action android:name="android.intent.action.MAIN" /> <category android:name="android.intent.category.LAUNCHER" /> @@ -36,14 +53,19 @@ <activity android:name=".PlayerActivity" - android:exported="false" - android:screenOrientation="portrait" /> + android:configChanges="orientation|screenSize|screenLayout|smallestScreenSize|keyboardHidden|uiMode" + android:exported="false" /> <service android:name=".RecordingService" android:exported="false" android:foregroundServiceType="microphone" /> + <service + android:name=".ModelDownloadService" + android:exported="false" + android:foregroundServiceType="dataSync" /> + <provider android:name="androidx.core.content.FileProvider" android:authorities="${applicationId}.fileprovider" diff --git a/app/src/main/java/com/tomcooks/reccoon/AudioCompressor.java b/app/src/main/java/com/tomcooks/reccoon/AudioCompressor.java new file mode 100644 index 0000000..168cf53 --- /dev/null +++ b/app/src/main/java/com/tomcooks/reccoon/AudioCompressor.java @@ -0,0 +1,139 @@ +package com.tomcooks.reccoon; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.media.MediaMuxer; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Transcodes a PCM WAV file into a small AAC-LC MP4/M4A file. + * + * <p>AAC-LC is used because every Android device ships an encoder for it, + * unlike Opus. At speech bitrates the result is roughly 10-20× smaller than the + * 44.1 kHz 16-bit WAV (for example ~22 MB per hour at 48 kbps mono instead of + * ~317 MB). The lossy step keeps enough bandwidth (16 kHz) for speech + * transcription; see {@link PcmAudioSource} for decoding it again. + */ +public final class AudioCompressor { + + private static final long TIMEOUT_US = 10_000; + private static final int CHUNK_FRAMES = 4096; + + private AudioCompressor() { + } + + /** @return the number of bytes written to {@code m4a}. */ + public static long compressToM4a(File wav, File m4a) throws IOException { + if (m4a.exists() && !m4a.delete()) { + throw new IOException("Cannot replace " + m4a); + } + try (BufferedInputStream in = + new BufferedInputStream(new FileInputStream(wav), 256 * 1024)) { + TranscriptionEngine.WavInfo info = TranscriptionEngine.readWavHeader(in); + if (info.bitsPerSample != 16 || info.audioFormat != 1) { + throw new IOException("Only 16-bit PCM WAV files can be compressed"); + } + encode(in, info, m4a); + } + return m4a.length(); + } + + private static void encode(InputStream pcm, TranscriptionEngine.WavInfo info, File m4a) + throws IOException { + int sampleRate = Math.max(1, info.sampleRate); + int channels = Math.max(1, info.channels); + + MediaFormat format = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, channels); + format.setInteger(MediaFormat.KEY_AAC_PROFILE, + MediaCodecInfo.CodecProfileLevel.AACObjectLC); + // Small but still clear for speech: 48 kbps per channel. + format.setInteger(MediaFormat.KEY_BIT_RATE, 48_000 * channels); + format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, CHUNK_FRAMES * channels * 2); + + MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC); + MediaMuxer muxer = new MediaMuxer(m4a.getAbsolutePath(), + MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + boolean muxerStarted = false; + boolean inputDone = false; + boolean outputDone = false; + int trackIndex = -1; + long presentationTimeUs = 0L; + long remaining = info.dataSize > 0 ? info.dataSize : Long.MAX_VALUE; + + try { + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + codec.start(); + + byte[] buffer = new byte[CHUNK_FRAMES * channels * 2]; + MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + + while (!outputDone) { + if (!inputDone) { + int inIndex = codec.dequeueInputBuffer(TIMEOUT_US); + if (inIndex >= 0) { + ByteBuffer input = codec.getInputBuffer(inIndex); + int capacity = input == null ? buffer.length : input.capacity(); + int toRead = (int) Math.min(Math.min(buffer.length, remaining), capacity); + int read = toRead <= 0 ? -1 : pcm.read(buffer, 0, toRead); + if (read < 0) { + codec.queueInputBuffer(inIndex, 0, 0, presentationTimeUs, + MediaCodec.BUFFER_FLAG_END_OF_STREAM); + inputDone = true; + } else { + remaining -= read; + input.clear(); + input.put(buffer, 0, read); + codec.queueInputBuffer(inIndex, 0, read, presentationTimeUs, 0); + long frames = read / (channels * 2L); + presentationTimeUs += frames * 1_000_000L / sampleRate; + } + } + } + + int outIndex = codec.dequeueOutputBuffer(bufferInfo, TIMEOUT_US); + if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + trackIndex = muxer.addTrack(codec.getOutputFormat()); + muxer.start(); + muxerStarted = true; + } else if (outIndex >= 0) { + ByteBuffer output = codec.getOutputBuffer(outIndex); + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + bufferInfo.size = 0; + } + if (bufferInfo.size > 0 && muxerStarted && output != null) { + output.position(bufferInfo.offset); + output.limit(bufferInfo.offset + bufferInfo.size); + muxer.writeSampleData(trackIndex, output, bufferInfo); + } + codec.releaseOutputBuffer(outIndex, false); + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true; + } + } + } + } finally { + try { + codec.stop(); + } catch (Exception ignored) { + // Ignore. + } + codec.release(); + if (muxerStarted) { + try { + muxer.stop(); + } catch (Exception ignored) { + // Ignore. + } + } + muxer.release(); + } + } +} diff --git a/app/src/main/java/com/wuhei/reccoon/Formats.java b/app/src/main/java/com/tomcooks/reccoon/Formats.java index 37e0909..338fd58 100644 --- a/app/src/main/java/com/wuhei/reccoon/Formats.java +++ b/app/src/main/java/com/tomcooks/reccoon/Formats.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import java.util.Locale; @@ -17,7 +17,7 @@ public final class Formats { if (core.startsWith("reccoon-")) { core = core.substring("reccoon-".length()); } - if (core.endsWith(".wav")) { + if (core.endsWith(".wav") || core.endsWith(".m4a")) { core = core.substring(0, core.length() - 4); } String[] parts = core.split("-"); diff --git a/app/src/main/java/com/wuhei/reccoon/InsetsHelper.java b/app/src/main/java/com/tomcooks/reccoon/InsetsHelper.java index 660fa64..27939be 100644 --- a/app/src/main/java/com/wuhei/reccoon/InsetsHelper.java +++ b/app/src/main/java/com/tomcooks/reccoon/InsetsHelper.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.view.View; diff --git a/app/src/main/java/com/wuhei/reccoon/LevelMeterView.java b/app/src/main/java/com/tomcooks/reccoon/LevelMeterView.java index d4b13e3..362ee6a 100644 --- a/app/src/main/java/com/wuhei/reccoon/LevelMeterView.java +++ b/app/src/main/java/com/tomcooks/reccoon/LevelMeterView.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Context; import android.graphics.Canvas; diff --git a/app/src/main/java/com/tomcooks/reccoon/LiveMonitor.java b/app/src/main/java/com/tomcooks/reccoon/LiveMonitor.java new file mode 100644 index 0000000..6366d84 --- /dev/null +++ b/app/src/main/java/com/tomcooks/reccoon/LiveMonitor.java @@ -0,0 +1,189 @@ +package com.tomcooks.reccoon; + +import android.content.Context; +import android.media.AudioAttributes; +import android.media.AudioDeviceInfo; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioTrack; + +import java.util.ArrayDeque; + +/** + * Plays captured PCM back to a chosen output device so the user can hear what is + * being recorded (live monitoring / sidetone). + * + * <p>The monitor is decoupled from the capture loop by a small bounded queue and + * a dedicated thread: if the output device is slow (for example Bluetooth A2DP + * with its inherent latency) chunks are dropped instead of stalling the + * recorder and corrupting the recording. + * + * <p>With the phone speaker as the output and the phone microphone as the input + * there will be feedback, so headphones are recommended. + */ +public final class LiveMonitor { + + private static final int MAX_QUEUED_CHUNKS = 8; + + private final AudioManager audioManager; + private final ArrayDeque<byte[]> queue = new ArrayDeque<>(); + + private AudioTrack track; + private Thread thread; + private volatile boolean running; + private int preferredDeviceId = -1; + + public LiveMonitor(Context context) { + audioManager = (AudioManager) context.getApplicationContext() + .getSystemService(Context.AUDIO_SERVICE); + } + + public void setPreferredDeviceId(int deviceId) { + this.preferredDeviceId = deviceId; + } + + public boolean isRunning() { + return running; + } + + /** @return true when the output could be opened and playback started. */ + public boolean start(int sampleRate, int channels) { + stop(); + int channelMask = channels > 1 + ? AudioFormat.CHANNEL_OUT_STEREO : AudioFormat.CHANNEL_OUT_MONO; + int minBuffer = AudioTrack.getMinBufferSize( + sampleRate, channelMask, AudioFormat.ENCODING_PCM_16BIT); + int target = sampleRate * channels * 2 / 5; // ~200 ms + int bufferSize = Math.max(Math.max(minBuffer, target), 1024); + + try { + track = new AudioTrack.Builder() + .setAudioAttributes(new AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build()) + .setAudioFormat(new AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(sampleRate) + .setChannelMask(channelMask) + .build()) + .setBufferSizeInBytes(bufferSize) + .setTransferMode(AudioTrack.MODE_STREAM) + .setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY) + .build(); + } catch (Exception e) { + track = null; + return false; + } + + if (track.getState() != AudioTrack.STATE_INITIALIZED) { + track.release(); + track = null; + return false; + } + applyPreferredDevice(); + try { + track.play(); + } catch (Exception e) { + track.release(); + track = null; + return false; + } + + running = true; + thread = new Thread(this::loop, "reccoon-monitor"); + thread.start(); + return true; + } + + /** Queues a chunk of PCM to play. Safe to call from the capture thread. */ + public void write(byte[] data, int length) { + if (!running || data == null || length <= 0) { + return; + } + byte[] copy = new byte[length]; + System.arraycopy(data, 0, copy, 0, length); + synchronized (queue) { + while (queue.size() >= MAX_QUEUED_CHUNKS) { + queue.pollFirst(); + } + queue.addLast(copy); + queue.notifyAll(); + } + } + + private void loop() { + while (running) { + byte[] chunk; + synchronized (queue) { + while (running && queue.isEmpty()) { + try { + queue.wait(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + chunk = queue.pollFirst(); + } + if (chunk == null) { + continue; + } + AudioTrack target = track; + if (target != null) { + try { + target.write(chunk, 0, chunk.length); + } catch (Exception ignored) { + // Drop this chunk and keep going. + } + } + } + } + + public void stop() { + running = false; + synchronized (queue) { + queue.clear(); + queue.notifyAll(); + } + Thread current = thread; + thread = null; + if (current != null) { + try { + current.join(500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + AudioTrack target = track; + track = null; + if (target != null) { + try { + target.stop(); + } catch (Exception ignored) { + // Ignore. + } + try { + target.release(); + } catch (Exception ignored) { + // Ignore. + } + } + } + + private void applyPreferredDevice() { + if (preferredDeviceId < 0 || audioManager == null || track == null) { + return; + } + try { + for (AudioDeviceInfo device : audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS)) { + if (device.getId() == preferredDeviceId) { + track.setPreferredDevice(device); + return; + } + } + } catch (Exception ignored) { + // Keep the default output. + } + } +} diff --git a/app/src/main/java/com/wuhei/reccoon/MainActivity.java b/app/src/main/java/com/tomcooks/reccoon/MainActivity.java index d11726f..b350965 100644 --- a/app/src/main/java/com/wuhei/reccoon/MainActivity.java +++ b/app/src/main/java/com/tomcooks/reccoon/MainActivity.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.Manifest; import android.animation.ObjectAnimator; @@ -9,6 +9,8 @@ import android.content.Context; import android.content.Intent; import android.content.SharedPreferences; import android.content.pm.PackageManager; +import android.media.AudioDeviceInfo; +import android.media.AudioManager; import android.media.MediaScannerConnection; import android.net.Uri; import android.os.Build; @@ -23,12 +25,14 @@ import android.text.InputType; import android.view.LayoutInflater; import android.view.View; import android.view.WindowManager; +import android.widget.ArrayAdapter; import android.widget.Button; import android.widget.EditText; import android.widget.FrameLayout; import android.widget.ImageButton; import android.widget.ImageView; import android.widget.LinearLayout; +import android.widget.Spinner; import android.widget.TextView; import android.widget.Toast; @@ -58,6 +62,17 @@ public class MainActivity extends AppCompatActivity { private static final String PREF_COUNTER = "counter"; private static final String PREF_SILENCE = "silence_notifications"; private static final String PREF_MONO = "mono_recording"; + private static final String PREF_COMPRESS = "compress_recordings"; + private static final String PREF_INPUT = "input_device"; + private static final String PREF_SWAP = "swap_channels"; + private static final String PREF_MONITOR = "live_monitor"; + private static final String PREF_OUTPUT = "output_device"; + + /** Spinner positions that are not a concrete {@link AudioDeviceInfo}. */ + private static final int INPUT_AUTO = -1; + private static final int INPUT_PHONE = -2; + private static final int OUTPUT_AUTO = -1; + private static final int OUTPUT_SPEAKER = -2; private ImageButton recordButton; private ImageView recIndicator; @@ -67,7 +82,15 @@ public class MainActivity extends AppCompatActivity { private WaveformView waveformView; private AppCompatCheckBox silenceCheckbox; private AppCompatCheckBox monoCheckbox; + private AppCompatCheckBox compressCheckbox; + private AppCompatCheckBox swapCheckbox; + private AppCompatCheckBox monitorCheckbox; + private Spinner inputSpinner; + private Spinner outputSpinner; + private final List<AudioDeviceInfo> inputDevices = new ArrayList<>(); + private final List<AudioDeviceInfo> outputDevices = new ArrayList<>(); private Button markButton; + private Button discardButton; private LinearLayout recordingsContainer; private TextView emptyText; private int savedInterruptionFilter = -1; @@ -96,7 +119,13 @@ public class MainActivity extends AppCompatActivity { waveformView = findViewById(R.id.waveformView); silenceCheckbox = findViewById(R.id.silenceCheckbox); monoCheckbox = findViewById(R.id.monoCheckbox); + compressCheckbox = findViewById(R.id.compressCheckbox); + swapCheckbox = findViewById(R.id.swapCheckbox); + monitorCheckbox = findViewById(R.id.monitorCheckbox); + inputSpinner = findViewById(R.id.inputSpinner); + outputSpinner = findViewById(R.id.outputSpinner); markButton = findViewById(R.id.markButton); + discardButton = findViewById(R.id.discardButton); recordingsContainer = findViewById(R.id.recordingsContainer); emptyText = findViewById(R.id.emptyText); @@ -114,7 +143,17 @@ public class MainActivity extends AppCompatActivity { setUpSilence(); setUpMono(); + setUpCompress(); + setUpSwap(); + setUpMonitor(); + populateInputDevices(); + populateOutputDevices(); markButton.setOnClickListener(v -> addMarker()); + discardButton.setOnClickListener(v -> { + if (recording && !busy) { + stopRecording(true); + } + }); InsetsHelper.apply(findViewById(R.id.root)); blinkAnimator = ObjectAnimator.ofFloat(recIndicator, View.ALPHA, 1f, 0.15f); @@ -128,6 +167,12 @@ public class MainActivity extends AppCompatActivity { @Override protected void onResume() { super.onResume(); + if (inputDevices.isEmpty()) { + populateInputDevices(); + } + if (outputDevices.isEmpty()) { + populateOutputDevices(); + } renderRecordings(); } @@ -174,7 +219,7 @@ public class MainActivity extends AppCompatActivity { return; } if (recording) { - stopRecordingAndSave(); + stopRecording(false); return; } List<String> missing = missingPermissions(); @@ -196,6 +241,12 @@ public class MainActivity extends AppCompatActivity { != PackageManager.PERMISSION_GRANTED) { missing.add(Manifest.permission.WRITE_EXTERNAL_STORAGE); } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S + && (selectedInputIsBluetooth() || selectedOutputIsBluetooth()) + && ContextCompat.checkSelfPermission(this, Manifest.permission.BLUETOOTH_CONNECT) + != PackageManager.PERMISSION_GRANTED) { + missing.add(Manifest.permission.BLUETOOTH_CONNECT); + } return missing; } @@ -209,6 +260,7 @@ public class MainActivity extends AppCompatActivity { } if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { + populateInputDevices(); startRecording(); } else { Toast.makeText(this, R.string.err_permission, Toast.LENGTH_LONG).show(); @@ -239,6 +291,316 @@ public class MainActivity extends AppCompatActivity { }); } + private void setUpCompress() { + SharedPreferences prefs = getSharedPreferences(PREFS, MODE_PRIVATE); + compressCheckbox.setChecked(prefs.getBoolean(PREF_COMPRESS, false)); + compressCheckbox.setOnCheckedChangeListener((button, checked) -> + getSharedPreferences(PREFS, MODE_PRIVATE) + .edit().putBoolean(PREF_COMPRESS, checked).apply()); + } + + private void setUpSwap() { + SharedPreferences prefs = getSharedPreferences(PREFS, MODE_PRIVATE); + swapCheckbox.setChecked(prefs.getBoolean(PREF_SWAP, false)); + swapCheckbox.setOnCheckedChangeListener((button, checked) -> + getSharedPreferences(PREFS, MODE_PRIVATE) + .edit().putBoolean(PREF_SWAP, checked).apply()); + } + + private void setUpMonitor() { + SharedPreferences prefs = getSharedPreferences(PREFS, MODE_PRIVATE); + monitorCheckbox.setChecked(prefs.getBoolean(PREF_MONITOR, false)); + monitorCheckbox.setOnCheckedChangeListener((button, checked) -> { + getSharedPreferences(PREFS, MODE_PRIVATE) + .edit().putBoolean(PREF_MONITOR, checked).apply(); + if (checked) { + Toast.makeText(this, R.string.monitor_warning, Toast.LENGTH_LONG).show(); + } + }); + } + + // --- Output source for live monitoring --------------------------------- + + private void populateOutputDevices() { + int selected = getSharedPreferences(PREFS, MODE_PRIVATE).getInt(PREF_OUTPUT, OUTPUT_AUTO); + outputDevices.clear(); + List<String> labels = new ArrayList<>(); + labels.add(getString(R.string.output_auto)); + labels.add(getString(R.string.output_speaker)); + + AudioManager manager = (AudioManager) getSystemService(Context.AUDIO_SERVICE); + if (manager != null) { + try { + for (AudioDeviceInfo device : manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS)) { + // The built-in speaker is already covered by the entry above. + if (device.getType() == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER) { + continue; + } + outputDevices.add(device); + labels.add(describeOutput(device)); + } + } catch (Exception ignored) { + // No device list available. + } + } + + ArrayAdapter<String> adapter = new ArrayAdapter<>(this, + android.R.layout.simple_spinner_item, labels); + adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); + outputSpinner.setAdapter(adapter); + + if (selected == OUTPUT_SPEAKER) { + outputSpinner.setSelection(1); + return; + } + for (int i = 0; i < outputDevices.size(); i++) { + if (outputDevices.get(i).getId() == selected) { + outputSpinner.setSelection(i + 2); + return; + } + } + outputSpinner.setSelection(0); + } + + /** Resolves the spinner to a concrete device id, or -1 for the default. */ + private int resolveOutputDeviceId() { + int position = outputSpinner.getSelectedItemPosition(); + if (position == 1) { + return findBuiltinSpeakerId(); + } + if (position >= 2 && position - 2 < outputDevices.size()) { + return outputDevices.get(position - 2).getId(); + } + return OUTPUT_AUTO; + } + + private int selectedOutputId() { + int position = outputSpinner.getSelectedItemPosition(); + if (position == 1) { + return OUTPUT_SPEAKER; + } + if (position >= 2 && position - 2 < outputDevices.size()) { + return outputDevices.get(position - 2).getId(); + } + return OUTPUT_AUTO; + } + + private boolean selectedOutputIsBluetooth() { + int position = outputSpinner.getSelectedItemPosition(); + return position >= 2 && position - 2 < outputDevices.size() + && isBluetoothOutput(outputDevices.get(position - 2)); + } + + private static boolean isBluetoothOutput(AudioDeviceInfo device) { + int type = device.getType(); + return type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP + || type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + || type == AudioDeviceInfo.TYPE_BLE_HEADSET + || type == AudioDeviceInfo.TYPE_BLE_SPEAKER; + } + + private int findBuiltinSpeakerId() { + AudioManager manager = (AudioManager) getSystemService(Context.AUDIO_SERVICE); + if (manager == null) { + return OUTPUT_AUTO; + } + try { + for (AudioDeviceInfo device : manager.getDevices(AudioManager.GET_DEVICES_OUTPUTS)) { + if (device.getType() == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER) { + return device.getId(); + } + } + } catch (Exception ignored) { + // Fall through to the default output. + } + return OUTPUT_AUTO; + } + + private static String describeOutput(AudioDeviceInfo device) { + String type; + switch (device.getType()) { + case AudioDeviceInfo.TYPE_BLUETOOTH_A2DP: + type = "Bluetooth"; + break; + case AudioDeviceInfo.TYPE_BLUETOOTH_SCO: + type = "Bluetooth (SCO)"; + break; + case AudioDeviceInfo.TYPE_BLE_HEADSET: + type = "Bluetooth LE headset"; + break; + case AudioDeviceInfo.TYPE_BLE_SPEAKER: + type = "Bluetooth LE speaker"; + break; + case AudioDeviceInfo.TYPE_WIRED_HEADPHONES: + type = "Wired headphones"; + break; + case AudioDeviceInfo.TYPE_WIRED_HEADSET: + type = "Wired headset"; + break; + case AudioDeviceInfo.TYPE_USB_DEVICE: + type = "USB device"; + break; + case AudioDeviceInfo.TYPE_USB_HEADSET: + type = "USB headset"; + break; + case AudioDeviceInfo.TYPE_BUILTIN_EARPIECE: + type = "Earpiece"; + break; + default: + type = "Output " + device.getType(); + break; + } + CharSequence name = device.getProductName(); + if (name != null && name.length() > 0) { + type += " \u2014 " + name; + } + return type; + } + + // --- Microphone source (BT-Mic-Force integration) --------------------- + + /** A resolved microphone choice to hand to {@link WavRecorder}. */ + private static final class InputChoice { + final int deviceId; + final boolean bluetoothSco; + + InputChoice(int deviceId, boolean bluetoothSco) { + this.deviceId = deviceId; + this.bluetoothSco = bluetoothSco; + } + } + + private void populateInputDevices() { + int selected = getSharedPreferences(PREFS, MODE_PRIVATE).getInt(PREF_INPUT, INPUT_AUTO); + inputDevices.clear(); + List<String> labels = new ArrayList<>(); + labels.add(getString(R.string.input_auto)); + labels.add(getString(R.string.input_phone)); + + AudioManager manager = (AudioManager) getSystemService(Context.AUDIO_SERVICE); + if (manager != null + && ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + == PackageManager.PERMISSION_GRANTED) { + try { + for (AudioDeviceInfo device : manager.getDevices(AudioManager.GET_DEVICES_INPUTS)) { + // The built-in mic is already covered by the "Phone mic" entry. + if (device.getType() == AudioDeviceInfo.TYPE_BUILTIN_MIC) { + continue; + } + inputDevices.add(device); + labels.add(describeInput(device)); + } + } catch (Exception ignored) { + // No device list available. + } + } + + ArrayAdapter<String> adapter = new ArrayAdapter<>(this, + android.R.layout.simple_spinner_item, labels); + adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); + inputSpinner.setAdapter(adapter); + + if (selected == INPUT_PHONE) { + inputSpinner.setSelection(1); + return; + } + for (int i = 0; i < inputDevices.size(); i++) { + if (inputDevices.get(i).getId() == selected) { + inputSpinner.setSelection(i + 2); + return; + } + } + inputSpinner.setSelection(0); + } + + private InputChoice resolveInput() { + int position = inputSpinner.getSelectedItemPosition(); + if (position == 1) { + return new InputChoice(findBuiltinMicId(), false); + } + if (position >= 2 && position - 2 < inputDevices.size()) { + AudioDeviceInfo device = inputDevices.get(position - 2); + return new InputChoice(device.getId(), isBluetoothInput(device)); + } + return new InputChoice(INPUT_AUTO, false); + } + + private int selectedInputId() { + int position = inputSpinner.getSelectedItemPosition(); + if (position == 1) { + return INPUT_PHONE; + } + if (position >= 2 && position - 2 < inputDevices.size()) { + return inputDevices.get(position - 2).getId(); + } + return INPUT_AUTO; + } + + private boolean selectedInputIsBluetooth() { + int position = inputSpinner.getSelectedItemPosition(); + return position >= 2 && position - 2 < inputDevices.size() + && isBluetoothInput(inputDevices.get(position - 2)); + } + + private int findBuiltinMicId() { + AudioManager manager = (AudioManager) getSystemService(Context.AUDIO_SERVICE); + if (manager == null) { + return INPUT_AUTO; + } + try { + for (AudioDeviceInfo device : manager.getDevices(AudioManager.GET_DEVICES_INPUTS)) { + if (device.getType() == AudioDeviceInfo.TYPE_BUILTIN_MIC) { + return device.getId(); + } + } + } catch (Exception ignored) { + // Fall through to auto. + } + return INPUT_AUTO; + } + + private static boolean isBluetoothInput(AudioDeviceInfo device) { + int type = device.getType(); + return type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + || type == AudioDeviceInfo.TYPE_BLE_HEADSET + || type == AudioDeviceInfo.TYPE_BLE_SPEAKER; + } + + private static String describeInput(AudioDeviceInfo device) { + String type; + switch (device.getType()) { + case AudioDeviceInfo.TYPE_BLUETOOTH_SCO: + type = "Bluetooth (SCO)"; + break; + case AudioDeviceInfo.TYPE_BLE_HEADSET: + type = "Bluetooth LE headset"; + break; + case AudioDeviceInfo.TYPE_BLE_SPEAKER: + type = "Bluetooth LE speaker"; + break; + case AudioDeviceInfo.TYPE_HEARING_AID: + type = "Hearing aid"; + break; + case AudioDeviceInfo.TYPE_USB_DEVICE: + type = "USB device"; + break; + case AudioDeviceInfo.TYPE_USB_HEADSET: + type = "USB headset"; + break; + case AudioDeviceInfo.TYPE_WIRED_HEADSET: + type = "Wired headset"; + break; + default: + type = "Input " + device.getType(); + break; + } + CharSequence name = device.getProductName(); + if (name != null && name.length() > 0) { + type += " \u2014 " + name; + } + return type; + } + private void addMarker() { if (!recording) { return; @@ -344,13 +706,26 @@ public class MainActivity extends AppCompatActivity { } private void startRecording() { + InputChoice input = resolveInput(); + getSharedPreferences(PREFS, MODE_PRIVATE) + .edit().putInt(PREF_INPUT, selectedInputId()).apply(); + recorder.setPreferredDeviceId(input.deviceId); + recorder.setBluetoothSco(input.bluetoothSco); + recorder.setSwapChannels(swapCheckbox.isChecked()); recorder.setForceMono(monoCheckbox.isChecked()); + recorder.setMonitoring(monitorCheckbox.isChecked()); + recorder.setMonitorDeviceId(resolveOutputDeviceId()); + getSharedPreferences(PREFS, MODE_PRIVATE) + .edit().putInt(PREF_OUTPUT, selectedOutputId()).apply(); try { recorder.start(); } catch (Exception e) { Toast.makeText(this, R.string.err_record, Toast.LENGTH_LONG).show(); return; } + if (monitorCheckbox.isChecked() && !recorder.isMonitoring()) { + Toast.makeText(this, R.string.err_monitor, Toast.LENGTH_LONG).show(); + } recording = true; startedAtMs = SystemClock.elapsedRealtime(); @@ -359,6 +734,7 @@ public class MainActivity extends AppCompatActivity { levelMeter.reset(); waveformView.reset(); markButton.setEnabled(true); + discardButton.setEnabled(true); applySilence(true); recordButton.setBackgroundResource(R.drawable.bg_record_active); recordButton.setContentDescription(getString(R.string.cd_stop)); @@ -379,7 +755,7 @@ public class MainActivity extends AppCompatActivity { handler.post(timerRunnable); } - private void stopRecordingAndSave() { + private void stopRecording(boolean discard) { recording = false; busy = true; @@ -387,6 +763,7 @@ public class MainActivity extends AppCompatActivity { levelMeter.reset(); waveformView.reset(); markButton.setEnabled(false); + discardButton.setEnabled(false); applySilence(false); long elapsed = SystemClock.elapsedRealtime() - startedAtMs; handler.removeCallbacks(timerRunnable); @@ -397,7 +774,8 @@ public class MainActivity extends AppCompatActivity { recordButton.setBackgroundResource(R.drawable.bg_record_idle); recordButton.setContentDescription(getString(R.string.cd_record)); recordButton.setEnabled(false); - statusText.setText(R.string.status_saving); + statusText.setText(discard ? R.string.status_discarding : R.string.status_saving); + final boolean compress = compressCheckbox.isChecked(); new Thread(() -> { File wav; @@ -411,9 +789,36 @@ public class MainActivity extends AppCompatActivity { } String savedName = null; - if (wav != null) { + boolean discarded = false; + if (wav != null && discard) { + //noinspection ResultOfMethodCallIgnored + wav.delete(); + discarded = true; + } else if (wav != null) { try { - savedName = saveToDownloads(wav); + if (compress) { + File m4a = new File(getCacheDir(), "reccoon_recording.m4a"); + try { + AudioCompressor.compressToM4a(wav, m4a); + savedName = saveToDownloads( + m4a, buildFileName(".m4a"), "audio/mp4"); + } catch (Exception compressionFailed) { + savedName = null; + } finally { + //noinspection ResultOfMethodCallIgnored + m4a.delete(); + } + if (savedName == null) { + // Never lose a recording just because compression failed. + savedName = saveToDownloads( + wav, buildFileName(".wav"), "audio/wav"); + } + //noinspection ResultOfMethodCallIgnored + wav.delete(); + } else { + savedName = saveToDownloads( + wav, buildFileName(".wav"), "audio/wav"); + } if (savedName != null && !markers.isEmpty()) { new MarkerStore(this).save(savedName, markers); } @@ -423,10 +828,14 @@ public class MainActivity extends AppCompatActivity { } final String result = savedName; + final boolean wasDiscarded = discarded; runOnUiThread(() -> { busy = false; recordButton.setEnabled(true); - if (result != null) { + if (wasDiscarded) { + statusText.setText(R.string.discarded); + Toast.makeText(this, R.string.discarded, Toast.LENGTH_SHORT).show(); + } else if (result != null) { statusText.setText(getString(R.string.saved_fmt, result)); Toast.makeText(this, getString(R.string.saved_fmt, result), Toast.LENGTH_LONG).show(); } else { @@ -438,21 +847,22 @@ public class MainActivity extends AppCompatActivity { }, "reccoon-save").start(); } - private String saveToDownloads(File wav) throws IOException { - String fileName = buildFileName(); + private String saveToDownloads(File file, String fileName, String mimeType) + throws IOException { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - saveViaMediaStore(wav, fileName); + saveViaMediaStore(file, fileName, mimeType); } else { - saveToPublicDownloads(wav, fileName); + saveToPublicDownloads(file, fileName, mimeType); } return fileName; } - private void saveViaMediaStore(File wav, String fileName) throws IOException { + private void saveViaMediaStore(File source, String fileName, String mimeType) + throws IOException { ContentResolver resolver = getContentResolver(); ContentValues values = new ContentValues(); values.put(MediaStore.MediaColumns.DISPLAY_NAME, fileName); - values.put(MediaStore.MediaColumns.MIME_TYPE, "audio/wav"); + values.put(MediaStore.MediaColumns.MIME_TYPE, mimeType); values.put(MediaStore.MediaColumns.RELATIVE_PATH, Environment.DIRECTORY_DOWNLOADS); values.put(MediaStore.MediaColumns.IS_PENDING, 1); @@ -465,7 +875,7 @@ public class MainActivity extends AppCompatActivity { if (out == null) { throw new IOException("MediaStore returned no output stream"); } - try (InputStream in = new FileInputStream(wav)) { + try (InputStream in = new FileInputStream(source)) { copy(in, out); } } @@ -476,29 +886,30 @@ public class MainActivity extends AppCompatActivity { } @SuppressWarnings("deprecation") - private void saveToPublicDownloads(File wav, String fileName) throws IOException { + private void saveToPublicDownloads(File source, String fileName, String mimeType) + throws IOException { File dir = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS); if (!dir.exists() && !dir.mkdirs()) { throw new IOException("Could not create the Downloads directory"); } File outFile = new File(dir, fileName); - try (InputStream in = new FileInputStream(wav); + try (InputStream in = new FileInputStream(source); OutputStream out = new FileOutputStream(outFile)) { copy(in, out); } MediaScannerConnection.scanFile( this, new String[]{outFile.getAbsolutePath()}, - new String[]{"audio/wav"}, + new String[]{mimeType}, null); } - private String buildFileName() { + private String buildFileName(String extension) { String stamp = new SimpleDateFormat("yyyy-MM-dd-HHmmss", Locale.US).format(new Date()); SharedPreferences prefs = getSharedPreferences(PREFS, MODE_PRIVATE); int next = prefs.getInt(PREF_COUNTER, 0) + 1; prefs.edit().putInt(PREF_COUNTER, next).apply(); - return String.format(Locale.US, "reccoon-%s-%05d.wav", stamp, next); + return String.format(Locale.US, "reccoon-%s-%05d%s", stamp, next, extension); } private static void copy(InputStream in, OutputStream out) throws IOException { diff --git a/app/src/main/java/com/wuhei/reccoon/Marker.java b/app/src/main/java/com/tomcooks/reccoon/Marker.java index d4a7ef5..a132cee 100644 --- a/app/src/main/java/com/wuhei/reccoon/Marker.java +++ b/app/src/main/java/com/tomcooks/reccoon/Marker.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; /** A user-placed marker inside a recording, stored as a WAV cue point. */ public final class Marker { diff --git a/app/src/main/java/com/wuhei/reccoon/MarkerStore.java b/app/src/main/java/com/tomcooks/reccoon/MarkerStore.java index bde7a24..a84ff87 100644 --- a/app/src/main/java/com/wuhei/reccoon/MarkerStore.java +++ b/app/src/main/java/com/tomcooks/reccoon/MarkerStore.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Context; diff --git a/app/src/main/java/com/tomcooks/reccoon/ModelDownloadService.java b/app/src/main/java/com/tomcooks/reccoon/ModelDownloadService.java new file mode 100644 index 0000000..1cea837 --- /dev/null +++ b/app/src/main/java/com/tomcooks/reccoon/ModelDownloadService.java @@ -0,0 +1,213 @@ +package com.tomcooks.reccoon; + +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.app.Service; +import android.content.Context; +import android.content.Intent; +import android.content.pm.ServiceInfo; +import android.os.Build; +import android.os.IBinder; +import android.os.PowerManager; + +import androidx.core.app.NotificationCompat; +import androidx.core.content.ContextCompat; + +/** + * Downloads a transcription model in a foreground service so the transfer keeps + * going when the screen turns off or the user switches to another app. A partial + * wake lock keeps the CPU up for the network I/O. + */ +public class ModelDownloadService extends Service { + + public static final String ACTION_START = "com.tomcooks.reccoon.action.DOWNLOAD_MODEL"; + public static final String EXTRA_MODEL = "model"; + + private static final String CHANNEL_ID = "reccoon_model_download"; + private static final int NOTIFICATION_ID = 2; + private static final long WAKE_LOCK_TIMEOUT_MS = 2 * 60 * 60 * 1000L; + + public interface Listener { + void onProgress(long downloaded, long total, String currentFile); + + void onComplete(); + + void onError(String message); + } + + private static volatile Listener sListener; + private static volatile boolean sRunning; + private static volatile long sDownloaded; + private static volatile long sTotal; + private static volatile String sCurrentFile = ""; + + public static void setListener(Listener listener) { + sListener = listener; + } + + public static boolean isRunning() { + return sRunning; + } + + public static long downloaded() { + return sDownloaded; + } + + public static long total() { + return sTotal; + } + + public static String currentFile() { + return sCurrentFile; + } + + public static void start(Context context, ModelRepository.Model model) { + Intent intent = new Intent(context, ModelDownloadService.class) + .setAction(ACTION_START) + .putExtra(EXTRA_MODEL, model.name()); + ContextCompat.startForegroundService(context, intent); + } + + private PowerManager.WakeLock wakeLock; + + @Override + public int onStartCommand(Intent intent, int flags, int startId) { + if (intent == null || !ACTION_START.equals(intent.getAction())) { + stopSelf(); + return START_NOT_STICKY; + } + ModelRepository.Model model; + try { + model = ModelRepository.Model.valueOf(intent.getStringExtra(EXTRA_MODEL)); + } catch (Exception e) { + stopSelf(); + return START_NOT_STICKY; + } + + startForegroundCompat(getString(R.string.status_downloading, "", 0)); + + if (sRunning) { + // A download is already in flight; keep the existing one. + return START_NOT_STICKY; + } + sRunning = true; + sDownloaded = 0L; + sTotal = model.totalBytes(); + sCurrentFile = ""; + acquireWakeLock(); + + new Thread(() -> { + Listener listener = sListener; + try { + ModelRepository repository = new ModelRepository(this); + repository.download(model, (downloaded, total, file) -> { + sDownloaded = downloaded; + sTotal = total; + sCurrentFile = file; + int permille = (int) (downloaded * 1000 / Math.max(1, total)); + updateNotification(getString( + R.string.status_downloading, file, permille / 10), permille); + Listener current = sListener; + if (current != null) { + current.onProgress(downloaded, total, file); + } + }); + if (listener != null) { + listener.onComplete(); + } + } catch (Exception e) { + if (listener != null) { + listener.onError(String.valueOf(e.getMessage())); + } + } finally { + sRunning = false; + releaseWakeLock(); + stopForeground(true); + stopSelf(); + } + }, "reccoon-model-download").start(); + + return START_NOT_STICKY; + } + + private void startForegroundCompat(String text) { + Notification notification = buildNotification(text, 0); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + startForeground(NOTIFICATION_ID, notification, + ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC); + } else { + startForeground(NOTIFICATION_ID, notification); + } + } + + private void updateNotification(String text, int permille) { + NotificationManager manager = + (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE); + if (manager != null) { + manager.notify(NOTIFICATION_ID, buildNotification(text, permille)); + } + } + + private Notification buildNotification(String text, int permille) { + createChannel(); + return new NotificationCompat.Builder(this, CHANNEL_ID) + .setContentTitle(getString(R.string.model_download_title)) + .setContentText(text) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setOngoing(true) + .setPriority(NotificationCompat.PRIORITY_LOW) + .setProgress(1000, permille, false) + .build(); + } + + private void createChannel() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + NotificationManager manager = getSystemService(NotificationManager.class); + if (manager != null && manager.getNotificationChannel(CHANNEL_ID) == null) { + NotificationChannel channel = new NotificationChannel( + CHANNEL_ID, + getString(R.string.model_download_channel), + NotificationManager.IMPORTANCE_LOW); + channel.setShowBadge(false); + manager.createNotificationChannel(channel); + } + } + } + + private void acquireWakeLock() { + try { + PowerManager power = (PowerManager) getSystemService(Context.POWER_SERVICE); + if (power != null) { + wakeLock = power.newWakeLock( + PowerManager.PARTIAL_WAKE_LOCK, "reccoon:model-download"); + wakeLock.acquire(WAKE_LOCK_TIMEOUT_MS); + } + } catch (Exception ignored) { + // Without the wake lock the download may pause when the screen is off. + } + } + + private void releaseWakeLock() { + try { + if (wakeLock != null && wakeLock.isHeld()) { + wakeLock.release(); + } + } catch (Exception ignored) { + // Ignore. + } + wakeLock = null; + } + + @Override + public void onDestroy() { + releaseWakeLock(); + sRunning = false; + super.onDestroy(); + } + + @Override + public IBinder onBind(Intent intent) { + return null; + } +} diff --git a/app/src/main/java/com/tomcooks/reccoon/ModelRepository.java b/app/src/main/java/com/tomcooks/reccoon/ModelRepository.java new file mode 100644 index 0000000..1d0e0be --- /dev/null +++ b/app/src/main/java/com/tomcooks/reccoon/ModelRepository.java @@ -0,0 +1,367 @@ +package com.tomcooks.reccoon; + +import android.content.Context; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +/** + * Downloads and stores the on-device speech recognition models. + * + * <p>Currently only NVIDIA Parakeet TDT 0.6B v3 (int8) is offered: a + * multilingual transducer (~670 MB) that was clearly the most accurate model in + * the evaluation harness. The enum leaves room for more models later. + * + * <p>A small Silero VAD model splits long recordings into transcribable chunks. + */ +public final class ModelRepository { + + private static final String HF_PARAKEET = + "https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8/resolve/main/"; + private static final String VAD_URL = + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx"; + + private static final long VAD_BYTES = 643_854L; + + /** A model the player can transcribe with. */ + public enum Model { + PARAKEET_V3("parakeet-v3", "Parakeet v3", + 652_184_281L, 11_845_275L, 6_355_277L, 93_939L); + + /** Folder under {@code filesDir/models} (or the external equivalent). */ + public final String dir; + public final String label; + final long encoderBytes; + final long decoderBytes; + final long joinerBytes; + final long tokensBytes; + + Model(String dir, String label, + long encoderBytes, long decoderBytes, long joinerBytes, long tokensBytes) { + this.dir = dir; + this.label = label; + this.encoderBytes = encoderBytes; + this.decoderBytes = decoderBytes; + this.joinerBytes = joinerBytes; + this.tokensBytes = tokensBytes; + } + + public long totalBytes() { + return encoderBytes + decoderBytes + joinerBytes + tokensBytes + VAD_BYTES; + } + } + + public interface Listener { + void onProgress(long downloaded, long total, String currentFile); + + default void onComplete() { + } + } + + private static final class RemoteFile { + final String url; + final File target; + final long expected; + + RemoteFile(String url, File target, long expected) { + this.url = url; + this.target = target; + this.expected = expected; + } + } + + private final File root; + + public ModelRepository(Context context) { + Context app = context.getApplicationContext(); + // Prefer the app-specific external folder (/sdcard/Android/data/<pkg>/files) + // so large models can be side-loaded with adb/scp or copied over USB. Fall + // back to internal storage when external storage is not mounted. + File external = app.getExternalFilesDir(null); + File base = external != null ? external : app.getFilesDir(); + root = new File(base, "models"); + migrateLegacyModels(base); + } + + /** + * Best-effort one-time migration from the old {@code com.tomcooks.reccoon} app + * id, so a re-branded install can reuse an already downloaded model instead + * of fetching it again. Android 11+ usually blocks access to another app's + * external directory; in that case this quietly does nothing and the model + * is downloaded again. + */ + private void migrateLegacyModels(File base) { + try { + File packageDir = base.getParentFile(); + File androidData = packageDir == null ? null : packageDir.getParentFile(); + if (androidData == null) { + return; + } + File legacy = new File( + new File(new File(androidData, "com.tomcooks.reccoon"), "files"), "models"); + if (legacy.isDirectory()) { + copyMissing(legacy, root); + } + } catch (Exception ignored) { + // Not readable (scoped storage); the model will simply be downloaded. + } + } + + private static void copyMissing(File from, File to) { + File[] children = from.listFiles(); + if (children == null) { + return; + } + if (!to.isDirectory() && !to.mkdirs()) { + return; + } + for (File child : children) { + File destination = new File(to, child.getName()); + if (child.isDirectory()) { + copyMissing(child, destination); + } else if (!destination.exists()) { + copyFile(child, destination); + } + } + } + + private static void copyFile(File from, File to) { + File temp = new File(to.getParentFile(), to.getName() + ".migrate"); + try (InputStream in = new FileInputStream(from); + FileOutputStream out = new FileOutputStream(temp)) { + byte[] buffer = new byte[256 * 1024]; + int read; + while ((read = in.read(buffer)) > 0) { + out.write(buffer, 0, read); + } + out.getFD().sync(); + //noinspection ResultOfMethodCallIgnored + temp.renameTo(to); + } catch (Exception ignored) { + //noinspection ResultOfMethodCallIgnored + temp.delete(); + } + } + + /** Folder that holds every model, for troubleshooting / manual side-loading. */ + public File modelsRoot() { + return root; + } + + public File vadModel() { + return new File(root, "silero_vad.onnx"); + } + + public File modelDir(Model model) { + return new File(root, model.dir); + } + + public File encoder(Model model) { + return new File(modelDir(model), "encoder.int8.onnx"); + } + + public File decoder(Model model) { + return new File(modelDir(model), "decoder.int8.onnx"); + } + + public File joiner(Model model) { + return new File(modelDir(model), "joiner.int8.onnx"); + } + + public File tokens(Model model) { + return new File(modelDir(model), "tokens.txt"); + } + + public boolean isReady(Model model) { + return isVadReady() && allComplete(files(model)); + } + + public boolean isVadReady() { + return isComplete(vadModel(), VAD_BYTES); + } + + private static boolean isComplete(File file, long expected) { + return file.isFile() && (expected <= 0 || file.length() == expected); + } + + private List<RemoteFile> files(Model model) { + List<RemoteFile> files = new ArrayList<>(); + files.add(new RemoteFile(VAD_URL, vadModel(), VAD_BYTES)); + files.add(new RemoteFile(HF_PARAKEET + "encoder.int8.onnx", + encoder(model), model.encoderBytes)); + files.add(new RemoteFile(HF_PARAKEET + "decoder.int8.onnx", + decoder(model), model.decoderBytes)); + files.add(new RemoteFile(HF_PARAKEET + "joiner.int8.onnx", + joiner(model), model.joinerBytes)); + files.add(new RemoteFile(HF_PARAKEET + "tokens.txt", + tokens(model), model.tokensBytes)); + return files; + } + + private boolean allComplete(List<RemoteFile> files) { + for (RemoteFile file : files) { + if (!isComplete(file.target, file.expected)) { + return false; + } + } + return true; + } + + /** + * Blocking. Call from a background thread. Progress is reported through + * {@code listener}. Throws on failure. + */ + public void download(Model model, Listener listener) throws IOException { + List<RemoteFile> files = files(model); + long total = 0; + long already = 0; + for (RemoteFile file : files) { + total += file.expected; + if (isComplete(file.target, file.expected)) { + already += file.expected; + } + } + + Counter counter = new Counter(already, total, listener); + counter.report(""); + for (RemoteFile file : files) { + fetch(file.url, file.target, file.expected, counter); + } + listener.onComplete(); + } + + private static final class Counter { + private final long already; + private final long total; + private final Listener listener; + private long doneInCurrent; + private String currentName = ""; + + Counter(long already, long total, Listener listener) { + this.already = already; + this.total = total; + this.listener = listener; + } + + void setCurrent(String name, long done) { + this.currentName = name; + this.doneInCurrent = done; + listener.onProgress(Math.min(already + done, total), total, currentName); + } + + void report(String name) { + listener.onProgress(Math.min(already + doneInCurrent, total), total, name); + } + } + + private static final int MAX_ATTEMPTS = 6; + + private static void fetch(String url, File target, long expected, Counter counter) + throws IOException { + if (isComplete(target, expected)) { + return; + } + File parent = target.getParentFile(); + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IOException("Cannot create " + parent); + } + File part = new File(parent, target.getName() + ".part"); + + IOException lastError = null; + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + downloadOnce(url, target, part, expected, counter); + return; + } catch (IOException e) { + lastError = e; + // Keep the .part file: the next attempt resumes with a Range request. + sleepQuietly(attempt); + } + } + throw lastError != null ? lastError : new IOException("Download failed: " + url); + } + + private static void downloadOnce(String url, File target, File part, long expected, + Counter counter) throws IOException { + long already = part.isFile() ? part.length() : 0L; + if (expected > 0 && already >= expected) { + if (already == expected) { + finish(part, target, expected); + return; + } + //noinspection ResultOfMethodCallIgnored + part.delete(); + already = 0L; + } + + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setInstanceFollowRedirects(true); + connection.setConnectTimeout(30_000); + connection.setReadTimeout(120_000); + connection.setRequestProperty("User-Agent", "RECCoon/1.0 (Android)"); + connection.setRequestProperty("Accept-Encoding", "identity"); + if (already > 0) { + connection.setRequestProperty("Range", "bytes=" + already + "-"); + } + try { + int code = connection.getResponseCode(); + if (code == HttpURLConnection.HTTP_OK) { + // The server ignored the Range request: start the file over. + already = 0L; + } else if (code != HttpURLConnection.HTTP_PARTIAL) { + throw new IOException("HTTP " + code + " for " + url); + } + + try (InputStream in = connection.getInputStream(); + FileOutputStream out = new FileOutputStream(part, already > 0)) { + byte[] buffer = new byte[256 * 1024]; + long done = already; + int read; + long lastReport = 0; + while ((read = in.read(buffer)) > 0) { + out.write(buffer, 0, read); + done += read; + long now = System.currentTimeMillis(); + if (now - lastReport > 200) { + lastReport = now; + counter.setCurrent(target.getName(), done); + } + } + out.getFD().sync(); + counter.setCurrent(target.getName(), done); + } + + finish(part, target, expected); + } finally { + connection.disconnect(); + } + } + + private static void finish(File part, File target, long expected) throws IOException { + if (expected > 0 && part.length() != expected) { + throw new IOException("Incomplete download for " + target.getName() + + " (have " + part.length() + " of " + expected + " bytes)"); + } + if (target.exists() && !target.delete()) { + throw new IOException("Cannot replace " + target); + } + if (!part.renameTo(target)) { + throw new IOException("Cannot move download into place: " + target); + } + } + + private static void sleepQuietly(int attempt) { + try { + Thread.sleep(Math.min(8_000L, 500L * (1L << Math.min(attempt, 4)))); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/app/src/main/java/com/tomcooks/reccoon/PcmAudioSource.java b/app/src/main/java/com/tomcooks/reccoon/PcmAudioSource.java new file mode 100644 index 0000000..9987d09 --- /dev/null +++ b/app/src/main/java/com/tomcooks/reccoon/PcmAudioSource.java @@ -0,0 +1,239 @@ +package com.tomcooks.reccoon; + +import android.content.Context; +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.net.Uri; + +import java.io.BufferedInputStream; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * A streaming source of 16-bit PCM samples. + * + * <p>Handles both the uncompressed WAV files that RECCoon records and the + * compressed files (AAC/M4A, Opus, ...) that the compact mode produces. WAV is + * parsed directly; everything else is decoded with {@link MediaCodec}. This is + * what lets transcription keep working on compressed recordings. + */ +public final class PcmAudioSource implements Closeable { + + private static final long CODEC_TIMEOUT_US = 10_000; + + /** Sample rate of the returned PCM. */ + public volatile int sampleRate; + /** Number of interleaved channels in the returned PCM. */ + public volatile int channels; + /** PCM size in bytes at {@link #sampleRate}, or 0 when unknown. */ + public final long totalBytes; + + private final InputStream wav; + private long wavRemaining; + + private final MediaExtractor extractor; + private final MediaCodec codec; + private boolean inputDone; + private boolean outputDone; + private byte[] pending; + private int pendingPos; + private int pendingLen; + + private PcmAudioSource(int sampleRate, int channels, long totalBytes, + InputStream wav, long wavRemaining, + MediaExtractor extractor, MediaCodec codec) { + this.sampleRate = sampleRate; + this.channels = channels; + this.totalBytes = totalBytes; + this.wav = wav; + this.wavRemaining = wavRemaining; + this.extractor = extractor; + this.codec = codec; + } + + /** + * Opens a WAV file when possible, otherwise falls back to a compressed + * decoder. Throws when neither path can read the URI. + */ + public static PcmAudioSource open(Context context, Uri uri) throws IOException { + InputStream raw = context.getContentResolver().openInputStream(uri); + if (raw == null) { + throw new IOException("Cannot open " + uri); + } + BufferedInputStream buffered = new BufferedInputStream(raw, 256 * 1024); + try { + TranscriptionEngine.WavInfo info = TranscriptionEngine.readWavHeader(buffered); + if (info.bitsPerSample != 16 || info.audioFormat != 1) { + throw new IOException("Only 16-bit PCM WAV files are supported"); + } + long remaining = info.dataSize > 0 ? info.dataSize : Long.MAX_VALUE; + return new PcmAudioSource(Math.max(1, info.sampleRate), Math.max(1, info.channels), + info.dataSize, buffered, remaining, null, null); + } catch (IOException notWav) { + closeQuietly(buffered); + } + return openCodec(context, uri); + } + + private static PcmAudioSource openCodec(Context context, Uri uri) throws IOException { + MediaExtractor extractor = new MediaExtractor(); + try { + extractor.setDataSource(context, uri, null); + } catch (IOException e) { + extractor.release(); + throw new IOException("Unsupported audio file: " + uri, e); + } + + int track = -1; + MediaFormat format = null; + for (int i = 0; i < extractor.getTrackCount(); i++) { + MediaFormat candidate = extractor.getTrackFormat(i); + String mime = candidate.getString(MediaFormat.KEY_MIME); + if (mime != null && mime.startsWith("audio/")) { + track = i; + format = candidate; + break; + } + } + if (track < 0 || format == null) { + extractor.release(); + throw new IOException("No audio track in " + uri); + } + extractor.selectTrack(track); + + String mime = format.getString(MediaFormat.KEY_MIME); + int sampleRate = format.containsKey(MediaFormat.KEY_SAMPLE_RATE) + ? format.getInteger(MediaFormat.KEY_SAMPLE_RATE) : 0; + int channels = format.containsKey(MediaFormat.KEY_CHANNEL_COUNT) + ? format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) : 1; + long durationUs = format.containsKey(MediaFormat.KEY_DURATION) + ? format.getLong(MediaFormat.KEY_DURATION) : 0L; + long totalBytes = sampleRate > 0 + ? durationUs * sampleRate * Math.max(1, channels) * 2 / 1_000_000L : 0L; + + MediaCodec codec; + try { + codec = MediaCodec.createDecoderByType(mime); + codec.configure(format, null, null, 0); + codec.start(); + } catch (Exception e) { + extractor.release(); + throw new IOException("Cannot decode " + mime, e); + } + return new PcmAudioSource(sampleRate, Math.max(1, channels), totalBytes, + null, 0L, extractor, codec); + } + + /** + * Reads up to {@code length} bytes of interleaved 16-bit PCM. Returns the + * number of bytes read, or -1 at end of stream. + */ + public int read(byte[] out, int offset, int length) throws IOException { + if (wav != null) { + if (wavRemaining <= 0) { + return -1; + } + int toRead = (int) Math.min(length, wavRemaining); + int total = 0; + while (total < toRead) { + int n = wav.read(out, offset + total, toRead - total); + if (n < 0) { + break; + } + total += n; + } + wavRemaining -= total; + return total == 0 ? -1 : total; + } + + int written = 0; + while (written < length) { + if (pending != null && pendingPos < pendingLen) { + int n = Math.min(length - written, pendingLen - pendingPos); + System.arraycopy(pending, pendingPos, out, offset + written, n); + pendingPos += n; + written += n; + continue; + } + if (outputDone) { + break; + } + decodeStep(); + } + return written == 0 ? -1 : written; + } + + private void decodeStep() throws IOException { + if (!inputDone) { + int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US); + if (inIndex >= 0) { + ByteBuffer input = codec.getInputBuffer(inIndex); + int size = input == null ? -1 : extractor.readSampleData(input, 0); + if (size < 0) { + codec.queueInputBuffer(inIndex, 0, 0, 0, + MediaCodec.BUFFER_FLAG_END_OF_STREAM); + inputDone = true; + } else { + codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0); + extractor.advance(); + } + } + } + + MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US); + if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + MediaFormat output = codec.getOutputFormat(); + if (output.containsKey(MediaFormat.KEY_SAMPLE_RATE)) { + sampleRate = output.getInteger(MediaFormat.KEY_SAMPLE_RATE); + } + if (output.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) { + channels = Math.max(1, output.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); + } + } else if (outIndex >= 0) { + if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true; + } + ByteBuffer output = codec.getOutputBuffer(outIndex); + if (info.size > 0 && output != null) { + byte[] chunk = new byte[info.size]; + output.position(info.offset); + output.limit(info.offset + info.size); + output.get(chunk); + pending = chunk; + pendingPos = 0; + pendingLen = info.size; + } + codec.releaseOutputBuffer(outIndex, false); + } + } + + @Override + public void close() { + closeQuietly(wav); + if (extractor != null) { + extractor.release(); + } + if (codec != null) { + try { + codec.stop(); + } catch (Exception ignored) { + // Ignore. + } + codec.release(); + } + } + + private static void closeQuietly(InputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException ignored) { + // Ignore. + } + } + } +} diff --git a/app/src/main/java/com/wuhei/reccoon/PlayerActivity.java b/app/src/main/java/com/tomcooks/reccoon/PlayerActivity.java index eb5b8ca..6d7d3f4 100644 --- a/app/src/main/java/com/wuhei/reccoon/PlayerActivity.java +++ b/app/src/main/java/com/tomcooks/reccoon/PlayerActivity.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Intent; import android.graphics.Color; @@ -31,6 +31,7 @@ import androidx.appcompat.app.AppCompatActivity; import java.io.File; import java.util.List; +import java.util.Locale; /** * Plays a saved recording with a scrubber and transcribes it on device with @@ -56,7 +57,6 @@ public class PlayerActivity extends AppCompatActivity { private Button playButton; private Button speedButton; private Button transcribeButton; - private Spinner languageSpinner; private Spinner modelSpinner; private ProgressBar progressBar; private LinearLayout transcriptContainer; @@ -72,6 +72,40 @@ public class PlayerActivity extends AppCompatActivity { private ModelRepository models; private TranscriptStore transcripts; private MarkerStore markerStore; + private ModelRepository.Model downloadingModel; + + private final ModelDownloadService.Listener downloadListener = + new ModelDownloadService.Listener() { + @Override + public void onProgress(long downloaded, long total, String file) { + runOnUiThread(() -> { + int permille = (int) (downloaded * 1000 / Math.max(1, total)); + progressBar.setIndeterminate(false); + progressBar.setProgress(permille); + statusText.setText(getString( + R.string.status_downloading, file, permille / 10)); + }); + } + + @Override + public void onComplete() { + runOnUiThread(() -> { + statusText.setText(R.string.status_model_ready); + if (downloadingModel != null) { + runTranscription(downloadingModel); + } + }); + } + + @Override + public void onError(String message) { + runOnUiThread(() -> { + transcribeButton.setEnabled(true); + progressBar.setVisibility(View.GONE); + statusText.setText(getString(R.string.err_download, message)); + }); + } + }; private final Runnable ticker = new Runnable() { @Override @@ -106,7 +140,6 @@ public class PlayerActivity extends AppCompatActivity { playButton = findViewById(R.id.playButton); speedButton = findViewById(R.id.speedButton); transcribeButton = findViewById(R.id.transcribeButton); - languageSpinner = findViewById(R.id.languageSpinner); modelSpinner = findViewById(R.id.modelSpinner); progressBar = findViewById(R.id.progressBar); transcriptContainer = findViewById(R.id.transcriptContainer); @@ -133,22 +166,13 @@ public class PlayerActivity extends AppCompatActivity { } private void setUpSpinners() { - ArrayAdapter<String> languages = new ArrayAdapter<>(this, - android.R.layout.simple_spinner_item, - new String[]{ - getString(R.string.lang_auto), - getString(R.string.lang_en), - getString(R.string.lang_it), - }); - languages.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); - languageSpinner.setAdapter(languages); - + ModelRepository.Model[] allModels = ModelRepository.Model.values(); + String[] modelLabels = new String[allModels.length]; + for (int i = 0; i < allModels.length; i++) { + modelLabels[i] = allModels[i].label; + } ArrayAdapter<String> modelAdapter = new ArrayAdapter<>(this, - android.R.layout.simple_spinner_item, - new String[]{ - getString(R.string.model_base), - getString(R.string.model_tiny), - }); + android.R.layout.simple_spinner_item, modelLabels); modelAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); modelSpinner.setAdapter(modelAdapter); } @@ -312,7 +336,8 @@ public class PlayerActivity extends AppCompatActivity { try { Uri shareUri = RecordingActions.shareableUri(this, recordingUri); Intent intent = new Intent(Intent.ACTION_SEND); - intent.setType("audio/wav"); + intent.setType(recordingName.toLowerCase(Locale.US).endsWith(".m4a") + ? "audio/mp4" : "audio/wav"); intent.putExtra(Intent.EXTRA_STREAM, shareUri); intent.addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION); startActivity(Intent.createChooser(intent, getString(R.string.action_share))); @@ -349,8 +374,10 @@ public class PlayerActivity extends AppCompatActivity { .setView(container) .setPositiveButton(R.string.action_rename, (dialog, which) -> { String requested = input.getText().toString().trim(); - if (!requested.endsWith(".wav")) { - requested = requested + ".wav"; + String lower = requested.toLowerCase(Locale.US); + if (!lower.endsWith(".wav") && !lower.endsWith(".m4a")) { + requested += recordingName.toLowerCase(Locale.US).endsWith(".m4a") + ? ".m4a" : ".wav"; } String renamed = RecordingActions.rename( this, recordingUri, recordingName, requested); @@ -389,58 +416,44 @@ public class PlayerActivity extends AppCompatActivity { // --- Transcription ---------------------------------------------------- private void onTranscribeClicked() { - ModelRepository.WhisperModel model = modelSpinner.getSelectedItemPosition() == 1 - ? ModelRepository.WhisperModel.TINY - : ModelRepository.WhisperModel.BASE; - String language = languageFor(languageSpinner.getSelectedItemPosition()); + ModelRepository.Model[] allModels = ModelRepository.Model.values(); + int position = modelSpinner.getSelectedItemPosition(); + if (position < 0 || position >= allModels.length) { + position = 0; + } + ModelRepository.Model model = allModels[position]; if (!models.isReady(model)) { - promptDownload(model, language); + promptDownload(model); } else { - runTranscription(model, language); + runTranscription(model); } } - private void promptDownload(ModelRepository.WhisperModel model, String language) { + private void promptDownload(ModelRepository.Model model) { new AlertDialog.Builder(this) .setTitle(R.string.model_download_title) .setMessage(getString(R.string.model_download_message, model.label, Formats.humanBytes(model.totalBytes()))) - .setPositiveButton(R.string.download, (dialog, which) -> runDownload(model, language)) + .setPositiveButton(R.string.download, (dialog, which) -> runDownload(model)) .setNegativeButton(R.string.cancel, null) .show(); } - private void runDownload(ModelRepository.WhisperModel model, String language) { + private void runDownload(ModelRepository.Model model) { + downloadingModel = model; transcribeButton.setEnabled(false); progressBar.setVisibility(View.VISIBLE); progressBar.setIndeterminate(false); progressBar.setProgress(0); statusText.setText(R.string.status_preparing); - - new Thread(() -> { - try { - models.download(model, (downloaded, total, file) -> runOnUiThread(() -> { - int permille = (int) (downloaded * 1000 / Math.max(1, total)); - progressBar.setProgress(permille); - statusText.setText(getString( - R.string.status_downloading, file, permille / 10)); - })); - runOnUiThread(() -> { - statusText.setText(R.string.status_model_ready); - runTranscription(model, language); - }); - } catch (Exception e) { - runOnUiThread(() -> { - transcribeButton.setEnabled(true); - progressBar.setVisibility(View.GONE); - statusText.setText(getString(R.string.err_download, String.valueOf(e.getMessage()))); - }); - } - }, "model-download").start(); + // The download runs in a foreground service so it survives the screen + // turning off or the app losing focus. + ModelDownloadService.setListener(downloadListener); + ModelDownloadService.start(this, model); } - private void runTranscription(ModelRepository.WhisperModel model, String language) { + private void runTranscription(ModelRepository.Model model) { cancelled = false; transcribeButton.setEnabled(false); transcriptContainer.removeAllViews(); @@ -452,7 +465,7 @@ public class PlayerActivity extends AppCompatActivity { TranscriptionEngine.Result result; try { TranscriptionEngine engine = new TranscriptionEngine(PlayerActivity.this); - result = engine.transcribe(recordingUri, model, language, + result = engine.transcribe(recordingUri, model, new TranscriptionEngine.Callback() { @Override public void onProgress(int percent, String message) { @@ -585,14 +598,19 @@ public class PlayerActivity extends AppCompatActivity { updateTimeLabel(); } - private static String languageFor(int position) { - switch (position) { - case 1: - return TranscriptionEngine.LANG_EN; - case 2: - return TranscriptionEngine.LANG_IT; - default: - return TranscriptionEngine.LANG_AUTO; + @Override + protected void onResume() { + super.onResume(); + if (ModelDownloadService.isRunning()) { + ModelDownloadService.setListener(downloadListener); + progressBar.setVisibility(View.VISIBLE); + progressBar.setIndeterminate(false); + long done = ModelDownloadService.downloaded(); + long total = ModelDownloadService.total(); + int permille = (int) (done * 1000 / Math.max(1, total)); + progressBar.setProgress(permille); + statusText.setText(getString(R.string.status_downloading, + ModelDownloadService.currentFile(), permille / 10)); } } @@ -614,6 +632,7 @@ public class PlayerActivity extends AppCompatActivity { @Override protected void onDestroy() { cancelled = true; + ModelDownloadService.setListener(null); handler.removeCallbacksAndMessages(null); if (player != null) { try { diff --git a/app/src/main/java/com/wuhei/reccoon/RecordingActions.java b/app/src/main/java/com/tomcooks/reccoon/RecordingActions.java index 4131181..3a110eb 100644 --- a/app/src/main/java/com/wuhei/reccoon/RecordingActions.java +++ b/app/src/main/java/com/tomcooks/reccoon/RecordingActions.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.ContentResolver; import android.content.ContentValues; @@ -41,10 +41,15 @@ public final class RecordingActions { /** @return the new file name, or null if renaming failed. */ public static String rename(Context context, Uri uri, String oldName, String newName) { - if (newName == null || newName.trim().isEmpty() || !newName.endsWith(".wav")) { + if (newName == null || newName.trim().isEmpty()) { return null; } newName = newName.trim(); + String lower = newName.toLowerCase(Locale.US); + if (!lower.endsWith(".wav") && !lower.endsWith(".m4a")) { + return null; + } + String mime = lower.endsWith(".m4a") ? "audio/mp4" : "audio/wav"; try { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && !"file".equals(uri.getScheme())) { ContentValues values = new ContentValues(); @@ -58,7 +63,7 @@ public final class RecordingActions { } MediaScannerConnection.scanFile(context, new String[]{newFile.getAbsolutePath()}, - new String[]{"audio/wav"}, null); + new String[]{mime}, null); } } catch (Exception e) { return null; @@ -83,7 +88,7 @@ public final class RecordingActions { if (result == null || result.segments.isEmpty()) { throw new IOException("Nothing to export"); } - String stem = recordingName.endsWith(".wav") + String stem = (recordingName.endsWith(".wav") || recordingName.endsWith(".m4a")) ? recordingName.substring(0, recordingName.length() - 4) : recordingName; writeToDownloads(context, stem + ".txt", "text/plain", result.asText()); diff --git a/app/src/main/java/com/wuhei/reccoon/RecordingService.java b/app/src/main/java/com/tomcooks/reccoon/RecordingService.java index 830c3bb..28bf803 100644 --- a/app/src/main/java/com/wuhei/reccoon/RecordingService.java +++ b/app/src/main/java/com/tomcooks/reccoon/RecordingService.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.app.Notification; import android.app.NotificationChannel; @@ -20,8 +20,8 @@ public class RecordingService extends Service { private static final String CHANNEL_ID = "reccoon_recording"; private static final int NOTIFICATION_ID = 1; - public static final String ACTION_START = "com.wuhei.reccoon.action.START"; - public static final String ACTION_STOP = "com.wuhei.reccoon.action.STOP"; + public static final String ACTION_START = "com.tomcooks.reccoon.action.START"; + public static final String ACTION_STOP = "com.tomcooks.reccoon.action.STOP"; @Override public int onStartCommand(Intent intent, int flags, int startId) { diff --git a/app/src/main/java/com/wuhei/reccoon/RecordingStore.java b/app/src/main/java/com/tomcooks/reccoon/RecordingStore.java index 12abb06..b7a1c84 100644 --- a/app/src/main/java/com/wuhei/reccoon/RecordingStore.java +++ b/app/src/main/java/com/tomcooks/reccoon/RecordingStore.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Context; import android.content.ContentUris; @@ -32,7 +32,7 @@ public final class RecordingStore { } private static final String PREFIX = "reccoon-"; - private static final String SUFFIX = ".wav"; + private static final String[] SUFFIXES = {".wav", ".m4a"}; private final Context context; @@ -40,6 +40,15 @@ public final class RecordingStore { this.context = context.getApplicationContext(); } + private static boolean hasKnownSuffix(String name) { + for (String suffix : SUFFIXES) { + if (name.endsWith(suffix)) { + return true; + } + } + return false; + } + public List<Recording> list() { List<Recording> recordings = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q ? listFromMediaStore() @@ -59,12 +68,23 @@ public final class RecordingStore { MediaStore.MediaColumns.SIZE, MediaStore.MediaColumns.DATE_MODIFIED, }; - String selection = MediaStore.MediaColumns.DISPLAY_NAME + " LIKE ? AND " - + MediaStore.MediaColumns.DISPLAY_NAME + " LIKE ?"; - String[] args = {PREFIX + "%", "%" + SUFFIX}; + StringBuilder selection = new StringBuilder(MediaStore.MediaColumns.DISPLAY_NAME) + .append(" LIKE ? AND ("); + for (int i = 0; i < SUFFIXES.length; i++) { + if (i > 0) { + selection.append(" OR "); + } + selection.append(MediaStore.MediaColumns.DISPLAY_NAME).append(" LIKE ?"); + } + selection.append(')'); + String[] args = new String[SUFFIXES.length + 1]; + args[0] = PREFIX + "%"; + for (int i = 0; i < SUFFIXES.length; i++) { + args[i + 1] = "%" + SUFFIXES[i]; + } try (Cursor cursor = context.getContentResolver().query( MediaStore.Downloads.EXTERNAL_CONTENT_URI, - projection, selection, args, null)) { + projection, selection.toString(), args, null)) { if (cursor == null) { return result; } @@ -92,7 +112,7 @@ public final class RecordingStore { private List<Recording> listFromDirectory() { List<Recording> result = new ArrayList<>(); File dir = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS); - File[] files = dir.listFiles((d, name) -> name.startsWith(PREFIX) && name.endsWith(SUFFIX)); + File[] files = dir.listFiles((d, name) -> name.startsWith(PREFIX) && hasKnownSuffix(name)); if (files == null) { return result; } diff --git a/app/src/main/java/com/wuhei/reccoon/StreamingResampler.java b/app/src/main/java/com/tomcooks/reccoon/StreamingResampler.java index beb372a..558f5ec 100644 --- a/app/src/main/java/com/wuhei/reccoon/StreamingResampler.java +++ b/app/src/main/java/com/tomcooks/reccoon/StreamingResampler.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import java.util.Arrays; diff --git a/app/src/main/java/com/wuhei/reccoon/TranscriptStore.java b/app/src/main/java/com/tomcooks/reccoon/TranscriptStore.java index 7ae336e..69d9439 100644 --- a/app/src/main/java/com/wuhei/reccoon/TranscriptStore.java +++ b/app/src/main/java/com/tomcooks/reccoon/TranscriptStore.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Context; diff --git a/app/src/main/java/com/wuhei/reccoon/TranscriptionEngine.java b/app/src/main/java/com/tomcooks/reccoon/TranscriptionEngine.java index 3579ab5..5483cbf 100644 --- a/app/src/main/java/com/wuhei/reccoon/TranscriptionEngine.java +++ b/app/src/main/java/com/tomcooks/reccoon/TranscriptionEngine.java @@ -1,6 +1,7 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Context; +import android.media.MediaMetadataRetriever; import android.net.Uri; import com.k2fsa.sherpa.onnx.FeatureConfig; @@ -9,7 +10,7 @@ import com.k2fsa.sherpa.onnx.OfflineRecognizer; import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; import com.k2fsa.sherpa.onnx.OfflineStream; -import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig; +import com.k2fsa.sherpa.onnx.OfflineTransducerModelConfig; import com.k2fsa.sherpa.onnx.SileroVadModelConfig; import com.k2fsa.sherpa.onnx.SpeechSegment; import com.k2fsa.sherpa.onnx.Vad; @@ -25,19 +26,15 @@ import java.util.ArrayList; import java.util.List; /** - * On-device transcription using sherpa-onnx with a multilingual Whisper model. + * On-device transcription using sherpa-onnx with the NVIDIA Parakeet TDT 0.6B + * v3 transducer (multilingual, auto language handling). * * <p>Long recordings are streamed, resampled to 16 kHz and segmented with * Silero VAD, then each detected speech segment is decoded. This keeps memory - * bounded regardless of the recording length (offline Whisper itself only - * handles up to 30 seconds at a time). + * bounded regardless of the recording length. */ public final class TranscriptionEngine { - public static final String LANG_AUTO = ""; - public static final String LANG_EN = "en"; - public static final String LANG_IT = "it"; - private static final int TARGET_RATE = 16_000; public static final class Word { @@ -98,59 +95,51 @@ public final class TranscriptionEngine { } public Result transcribe(Uri wavUri, - ModelRepository.WhisperModel model, - String language, + ModelRepository.Model model, Callback callback) throws IOException { if (!models.isReady(model)) { throw new IOException("Model is not downloaded"); } - OfflineRecognizer recognizer = buildRecognizer(model, language); + OfflineRecognizer recognizer = buildRecognizer(model); Vad vad = buildVad(); Result result = new Result(); - result.language = language; - - try (InputStream raw = context.getContentResolver().openInputStream(wavUri)) { - if (raw == null) { - throw new IOException("Cannot open " + wavUri); - } - BufferedInputStream buffered = new BufferedInputStream(raw, 256 * 1024); - WavInfo info = readWavHeader(buffered); - if (info.bitsPerSample != 16 || info.audioFormat != 1) { - throw new IOException("Only 16-bit PCM WAV files are supported"); - } - - StreamingResampler resampler = new StreamingResampler(info.sampleRate, TARGET_RATE); - long totalPcm16k = estimatePcm16kSamples(info); + result.language = ""; + + // PcmAudioSource reads uncompressed WAV directly and decodes compressed + // recordings (AAC/M4A, ...) through MediaCodec, so compact mode keeps + // working here. + try (PcmAudioSource source = PcmAudioSource.open(context, wavUri)) { + int channelCount = source.channels; + StreamingResampler resampler = + new StreamingResampler(source.sampleRate, TARGET_RATE); + long totalPcm16k = estimatePcm16kSamples(source); long processed = 0; - int frameBytes = info.channels * 2; + int frameBytes = channelCount * 2; byte[] readBuffer = new byte[frameBytes * 8192]; - long remaining = info.dataSize > 0 ? info.dataSize : Long.MAX_VALUE; - while (remaining > 0) { + while (true) { if (callback != null && callback.isCancelled()) { return null; } - int toRead = (int) Math.min(readBuffer.length, remaining); - int read = buffered.read(readBuffer, 0, toRead); + int read = source.read(readBuffer, 0, readBuffer.length); if (read <= 0) { break; } - remaining -= read; int frames = read / frameBytes; float[] mono = new float[frames]; for (int i = 0; i < frames; i++) { int offset = i * frameBytes; - if (info.channels == 1) { + if (channelCount == 1) { mono[i] = pcm16(readBuffer, offset) / 32768f; } else { int sum = 0; - for (int c = 0; c < info.channels; c++) { + for (int c = 0; c < channelCount; c++) { sum += pcm16(readBuffer, offset + c * 2); } - mono[i] = sum / (float) info.channels / 32768f; + mono[i] = sum / (float) channelCount / 32768f; } } @@ -252,28 +241,24 @@ public final class TranscriptionEngine { } } - private OfflineRecognizer buildRecognizer(ModelRepository.WhisperModel model, String language) { + private OfflineRecognizer buildRecognizer(ModelRepository.Model model) { FeatureConfig feat = new FeatureConfig(); feat.setSampleRate(TARGET_RATE); feat.setFeatureDim(80); feat.setDither(0f); - OfflineWhisperModelConfig whisper = new OfflineWhisperModelConfig(); - whisper.setEncoder(models.encoder(model).getAbsolutePath()); - whisper.setDecoder(models.decoder(model).getAbsolutePath()); - whisper.setLanguage(language == null ? "" : language); - whisper.setTask("transcribe"); - whisper.setTailPaddings(1000); - whisper.setEnableTokenTimestamps(true); - whisper.setEnableSegmentTimestamps(false); + OfflineTransducerModelConfig transducer = new OfflineTransducerModelConfig(); + transducer.setEncoder(models.encoder(model).getAbsolutePath()); + transducer.setDecoder(models.decoder(model).getAbsolutePath()); + transducer.setJoiner(models.joiner(model).getAbsolutePath()); OfflineModelConfig modelConfig = new OfflineModelConfig(); - modelConfig.setWhisper(whisper); + modelConfig.setTransducer(transducer); + modelConfig.setModelType("nemo_transducer"); modelConfig.setTokens(models.tokens(model).getAbsolutePath()); modelConfig.setNumThreads(Math.max(2, Runtime.getRuntime().availableProcessors() / 2)); modelConfig.setDebug(false); modelConfig.setProvider("cpu"); - modelConfig.setModelType("whisper"); OfflineRecognizerConfig config = new OfflineRecognizerConfig(); config.setFeatConfig(feat); @@ -365,26 +350,36 @@ public final class TranscriptionEngine { public static long durationMillis(Context context, Uri uri) { try (InputStream in = context.getContentResolver().openInputStream(uri)) { - if (in == null) { - return 0; + if (in != null) { + WavInfo info = readWavHeader(new BufferedInputStream(in)); + return info.durationMillis(); } - WavInfo info = readWavHeader(new BufferedInputStream(in)); - return info.durationMillis(); + } catch (Exception ignored) { + // Not a WAV file; try the platform decoder below. + } + MediaMetadataRetriever retriever = new MediaMetadataRetriever(); + try { + retriever.setDataSource(context, uri); + String value = retriever.extractMetadata( + MediaMetadataRetriever.METADATA_KEY_DURATION); + return value == null ? 0 : Long.parseLong(value); } catch (Exception e) { return 0; + } finally { + try { + retriever.release(); + } catch (Exception ignored) { + // Ignore. + } } } - private static long estimatePcm16kSamples(WavInfo info) { - if (info.dataSize <= 0 || info.sampleRate <= 0) { - return 0; - } - int frameBytes = info.channels * (info.bitsPerSample / 8); - if (frameBytes <= 0) { + private static long estimatePcm16kSamples(PcmAudioSource source) { + if (source.totalBytes <= 0 || source.sampleRate <= 0 || source.channels <= 0) { return 0; } - long frames = info.dataSize / frameBytes; - return frames * TARGET_RATE / info.sampleRate; + long frames = source.totalBytes / (source.channels * 2L); + return frames * TARGET_RATE / source.sampleRate; } private static int pcm16(byte[] data, int offset) { diff --git a/app/src/main/java/com/wuhei/reccoon/WavMarkers.java b/app/src/main/java/com/tomcooks/reccoon/WavMarkers.java index 1f55e96..7ef8425 100644 --- a/app/src/main/java/com/wuhei/reccoon/WavMarkers.java +++ b/app/src/main/java/com/tomcooks/reccoon/WavMarkers.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Context; import android.net.Uri; diff --git a/app/src/main/java/com/wuhei/reccoon/WavRecorder.java b/app/src/main/java/com/tomcooks/reccoon/WavRecorder.java index 8279847..0171a74 100644 --- a/app/src/main/java/com/wuhei/reccoon/WavRecorder.java +++ b/app/src/main/java/com/tomcooks/reccoon/WavRecorder.java @@ -1,7 +1,9 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Context; +import android.media.AudioDeviceInfo; import android.media.AudioFormat; +import android.media.AudioManager; import android.media.AudioRecord; import android.media.MediaRecorder; import android.os.SystemClock; @@ -51,6 +53,7 @@ public class WavRecorder { } private final Context context; + private final AudioManager audioManager; private final List<Marker> markers = new ArrayList<>(); private LevelListener levelListener; @@ -61,9 +64,18 @@ public class WavRecorder { private volatile long pcmBytes; private int channels = 2; private boolean forceMono; + private int preferredDeviceId = -1; + private boolean bluetoothSco; + private boolean scoActive; + private boolean swapChannels; + private boolean monitoring; + private int monitorDeviceId = -1; + private final LiveMonitor monitor; public WavRecorder(Context context) { this.context = context.getApplicationContext(); + this.audioManager = (AudioManager) this.context.getSystemService(Context.AUDIO_SERVICE); + this.monitor = new LiveMonitor(this.context); } public void setLevelListener(LevelListener listener) { @@ -75,6 +87,49 @@ public class WavRecorder { this.forceMono = forceMono; } + /** + * Forces a specific input device, or -1 to let Android pick. Mirrors the + * BT-Mic-Force behaviour: the id comes from + * {@link AudioManager#getDevices(int)}. + */ + public void setPreferredDeviceId(int deviceId) { + this.preferredDeviceId = deviceId; + } + + /** + * When true, route audio through Bluetooth SCO (headset microphone) before + * opening the recorder. Call {@link #setPreferredDeviceId(int)} as well to + * pick the exact headset when several are connected. + */ + public void setBluetoothSco(boolean bluetoothSco) { + this.bluetoothSco = bluetoothSco; + } + + /** + * When true, exchange the two stereo channels before writing. Some devices + * report the microphones in the opposite order once the phone is turned to + * landscape; swapping makes both the meters and the saved file agree with + * what the user hears on their left/right. + */ + public void setSwapChannels(boolean swapChannels) { + this.swapChannels = swapChannels; + } + + /** When true, play the captured audio back through {@link LiveMonitor}. */ + public void setMonitoring(boolean monitoring) { + this.monitoring = monitoring; + } + + /** Output device for live monitoring, or -1 for the system default. */ + public void setMonitorDeviceId(int deviceId) { + this.monitorDeviceId = deviceId; + } + + /** True while live monitoring is actually running. */ + public boolean isMonitoring() { + return monitoring && monitor.isRunning(); + } + /** Effective number of channels used by the last/current recording. */ public int getChannels() { return channels; @@ -101,6 +156,11 @@ public class WavRecorder { recording = true; audioRecord.startRecording(); + if (monitoring) { + monitor.setPreferredDeviceId(monitorDeviceId); + monitor.start(SAMPLE_RATE, channels); + } + final AudioRecord record = audioRecord; final int channelCount = channels; final int frameBytes = channelCount * 2; @@ -115,7 +175,11 @@ public class WavRecorder { while (recording) { int read = record.read(buffer, 0, buffer.length); if (read > 0) { + maybeSwapChannels(buffer, read, channelCount); out.write(buffer, 0, read); + if (monitoring) { + monitor.write(buffer, read); + } pcmBytes += read; long now = SystemClock.uptimeMillis(); @@ -135,6 +199,25 @@ public class WavRecorder { return tempFile; } + /** Exchanges the two 16-bit channels in-place when the swap option is on. */ + private void maybeSwapChannels(byte[] buffer, int length, int channelCount) { + if (!swapChannels || channelCount != 2) { + return; + } + int frames = length / 4; + for (int i = 0; i < frames; i++) { + int offset = i * 4; + byte leftLow = buffer[offset]; + byte leftHigh = buffer[offset + 1]; + byte rightLow = buffer[offset + 2]; + byte rightHigh = buffer[offset + 3]; + buffer[offset] = rightLow; + buffer[offset + 1] = rightHigh; + buffer[offset + 2] = leftLow; + buffer[offset + 3] = leftHigh; + } + } + private void reportLevels(byte[] buffer, int length, int channelCount) { int frameBytes = channelCount * 2; int frames = length / frameBytes; @@ -253,6 +336,8 @@ public class WavRecorder { audioRecord = null; } writerThread = null; + monitor.stop(); + stopSco(); writeHeader(); writeMarkers(); @@ -264,6 +349,10 @@ public class WavRecorder { } private AudioRecord createAudioRecord() { + startScoIfNeeded(); + int source = bluetoothSco + ? MediaRecorder.AudioSource.VOICE_COMMUNICATION + : MediaRecorder.AudioSource.MIC; int[] candidates = forceMono ? new int[]{AudioFormat.CHANNEL_IN_MONO} : new int[]{AudioFormat.CHANNEL_IN_STEREO, AudioFormat.CHANNEL_IN_MONO}; @@ -276,20 +365,66 @@ public class WavRecorder { int count = channelConfig == AudioFormat.CHANNEL_IN_STEREO ? 2 : 1; int bufferSize = Math.max(minBuffer, SAMPLE_RATE * count); AudioRecord candidate = new AudioRecord( - MediaRecorder.AudioSource.MIC, + source, SAMPLE_RATE, channelConfig, AudioFormat.ENCODING_PCM_16BIT, bufferSize); if (candidate.getState() == AudioRecord.STATE_INITIALIZED) { + applyPreferredDevice(candidate); channels = count; return candidate; } candidate.release(); } + stopSco(); return null; } + private void startScoIfNeeded() { + if (!bluetoothSco || audioManager == null) { + return; + } + try { + audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION); + audioManager.startBluetoothSco(); + audioManager.setBluetoothScoOn(true); + scoActive = true; + } catch (Exception ignored) { + // Fall back to the default route below. + } + } + + private void applyPreferredDevice(AudioRecord record) { + if (preferredDeviceId < 0 || audioManager == null) { + return; + } + try { + for (AudioDeviceInfo device : audioManager.getDevices(AudioManager.GET_DEVICES_INPUTS)) { + if (device.getId() == preferredDeviceId) { + record.setPreferredDevice(device); + return; + } + } + } catch (Exception ignored) { + // Keep the default device. + } + } + + private void stopSco() { + if (!scoActive || audioManager == null) { + return; + } + scoActive = false; + try { + audioManager.stopBluetoothSco(); + audioManager.setBluetoothScoOn(false); + audioManager.setMode(AudioManager.MODE_NORMAL); + } catch (Exception ignored) { + // Ignore. + } + } + private static int channelConfigFor(int channelCount) { return channelCount > 1 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO; } diff --git a/app/src/main/java/com/wuhei/reccoon/WaveformView.java b/app/src/main/java/com/tomcooks/reccoon/WaveformView.java index de62639..413a825 100644 --- a/app/src/main/java/com/wuhei/reccoon/WaveformView.java +++ b/app/src/main/java/com/tomcooks/reccoon/WaveformView.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import android.content.Context; import android.graphics.Canvas; diff --git a/app/src/main/java/com/wuhei/reccoon/ModelRepository.java b/app/src/main/java/com/wuhei/reccoon/ModelRepository.java deleted file mode 100644 index cbf25c2..0000000 --- a/app/src/main/java/com/wuhei/reccoon/ModelRepository.java +++ /dev/null @@ -1,206 +0,0 @@ -package com.wuhei.reccoon; - -import android.content.Context; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; - -/** - * Downloads and stores the on-device speech recognition models. - * - * <p>Uses multilingual Whisper (tiny or base, int8) from the sherpa-onnx model - * zoo. The multilingual variants support both English and Italian. A small - * Silero VAD model is used to split long recordings into transcribable chunks. - */ -public final class ModelRepository { - - private static final String HF_BASE = - "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-"; - private static final String VAD_URL = - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx"; - - public enum WhisperModel { - TINY("tiny", "Tiny", 12_937_772L, 89_855_401L), - BASE("base", "Base", 29_120_534L, 130_672_026L); - - public final String id; - public final String label; - public final long encoderBytes; - public final long decoderBytes; - - WhisperModel(String id, String label, long encoderBytes, long decoderBytes) { - this.id = id; - this.label = label; - this.encoderBytes = encoderBytes; - this.decoderBytes = decoderBytes; - } - - public long totalBytes() { - return encoderBytes + decoderBytes + TOKENS_BYTES + VAD_BYTES; - } - } - - private static final long TOKENS_BYTES = 816_730L; - private static final long VAD_BYTES = 643_854L; - - public interface Listener { - void onProgress(long downloaded, long total, String currentFile); - - default void onComplete() { - } - } - - private final File root; - - public ModelRepository(Context context) { - root = new File(context.getApplicationContext().getFilesDir(), "models"); - } - - public File vadModel() { - return new File(root, "silero_vad.onnx"); - } - - public File encoder(WhisperModel model) { - return new File(modelDir(model), "encoder.int8.onnx"); - } - - public File decoder(WhisperModel model) { - return new File(modelDir(model), "decoder.int8.onnx"); - } - - public File tokens(WhisperModel model) { - return new File(modelDir(model), "tokens.txt"); - } - - private File modelDir(WhisperModel model) { - return new File(root, "whisper-" + model.id); - } - - public boolean isReady(WhisperModel model) { - return isComplete(vadModel(), VAD_BYTES) - && isComplete(encoder(model), model.encoderBytes) - && isComplete(decoder(model), model.decoderBytes) - && isComplete(tokens(model), TOKENS_BYTES); - } - - public boolean isVadReady() { - return isComplete(vadModel(), VAD_BYTES); - } - - private static boolean isComplete(File file, long expected) { - return file.isFile() && (expected <= 0 || file.length() == expected); - } - - /** - * Blocking. Call from a background thread. Progress is reported through - * {@code listener}. Throws on failure. - */ - public void download(WhisperModel model, Listener listener) throws IOException { - long total = model.totalBytes(); - long already = 0; - if (isComplete(vadModel(), VAD_BYTES)) already += VAD_BYTES; - if (isComplete(encoder(model), model.encoderBytes)) already += model.encoderBytes; - if (isComplete(decoder(model), model.decoderBytes)) already += model.decoderBytes; - if (isComplete(tokens(model), TOKENS_BYTES)) already += TOKENS_BYTES; - - Counter counter = new Counter(already, total, listener); - counter.report(""); - - fetch(VAD_URL, vadModel(), VAD_BYTES, counter); - String base = HF_BASE + model.id + "/resolve/main/" + model.id; - fetch(base + "-encoder.int8.onnx", encoder(model), model.encoderBytes, counter); - fetch(base + "-decoder.int8.onnx", decoder(model), model.decoderBytes, counter); - fetch(base + "-tokens.txt", tokens(model), TOKENS_BYTES, counter); - - listener.onComplete(); - } - - private static final class Counter { - private final long already; - private final long total; - private final Listener listener; - private long doneInCurrent; - private String currentName = ""; - - Counter(long already, long total, Listener listener) { - this.already = already; - this.total = total; - this.listener = listener; - } - - void setCurrent(String name, long done) { - this.currentName = name; - this.doneInCurrent = done; - listener.onProgress(Math.min(already + done, total), total, currentName); - } - - void report(String name) { - listener.onProgress(Math.min(already + doneInCurrent, total), total, name); - } - } - - private static void fetch(String url, File target, long expected, Counter counter) - throws IOException { - if (isComplete(target, expected)) { - return; - } - File parent = target.getParentFile(); - if (parent != null && !parent.exists() && !parent.mkdirs()) { - throw new IOException("Cannot create " + parent); - } - File part = new File(parent, target.getName() + ".part"); - - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); - connection.setInstanceFollowRedirects(true); - connection.setConnectTimeout(30_000); - connection.setReadTimeout(120_000); - connection.setRequestProperty("User-Agent", "RECCoon/1.0 (Android)"); - try { - int code = connection.getResponseCode(); - if (code != HttpURLConnection.HTTP_OK) { - throw new IOException("HTTP " + code + " for " + url); - } - long length = connection.getContentLengthLong(); - long thisTotal = length > 0 ? length : expected; - - try (InputStream in = connection.getInputStream(); - FileOutputStream out = new FileOutputStream(part)) { - byte[] buffer = new byte[256 * 1024]; - long done = 0; - int read; - long lastReport = 0; - while ((read = in.read(buffer)) > 0) { - out.write(buffer, 0, read); - done += read; - long now = System.currentTimeMillis(); - if (now - lastReport > 200) { - lastReport = now; - counter.setCurrent(target.getName(), done); - } - } - out.getFD().sync(); - counter.setCurrent(target.getName(), done); - } - - if (expected > 0 && part.length() != expected) { - //noinspection ResultOfMethodCallIgnored - part.delete(); - throw new IOException( - "Corrupt download for " + target.getName() - + " (expected " + expected + " bytes, got " + part.length() + ")"); - } - if (target.exists() && !target.delete()) { - throw new IOException("Cannot replace " + target); - } - if (!part.renameTo(target)) { - throw new IOException("Cannot move download into place: " + target); - } - } finally { - connection.disconnect(); - } - } -} diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index e919117..2f3a1ac 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -65,27 +65,113 @@ android:scaleType="centerInside" android:src="@android:color/transparent" /> - <com.wuhei.reccoon.WaveformView + <androidx.appcompat.widget.AppCompatCheckBox + android:id="@+id/compressCheckbox" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:layout_gravity="center_horizontal" + android:layout_marginTop="4dp" + android:contentDescription="@string/compress_recording" + android:text="@string/compress_short" + android:textColor="@color/text_secondary" + android:textSize="13sp" /> + + <LinearLayout + android:id="@+id/inputRow" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:layout_gravity="center_horizontal" + android:layout_marginTop="4dp" + android:gravity="center_vertical" + android:orientation="horizontal"> + + <TextView + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="@string/input_label" + android:textColor="@color/text_secondary" + android:textSize="13sp" /> + + <Spinner + android:id="@+id/inputSpinner" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:layout_marginStart="8dp" + android:minWidth="180dp" /> + </LinearLayout> + + <LinearLayout + android:id="@+id/outputRow" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:layout_gravity="center_horizontal" + android:layout_marginTop="4dp" + android:gravity="center_vertical" + android:orientation="horizontal"> + + <TextView + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="@string/output_label" + android:textColor="@color/text_secondary" + android:textSize="13sp" /> + + <Spinner + android:id="@+id/outputSpinner" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:layout_marginStart="8dp" + android:minWidth="180dp" /> + </LinearLayout> + + <androidx.appcompat.widget.AppCompatCheckBox + android:id="@+id/monitorCheckbox" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:layout_gravity="center_horizontal" + android:layout_marginTop="2dp" + android:contentDescription="@string/monitor_recording" + android:text="@string/monitor_short" + android:textColor="@color/text_secondary" + android:textSize="13sp" /> + + <com.tomcooks.reccoon.WaveformView android:id="@+id/waveformView" android:layout_width="match_parent" android:layout_height="64dp" android:layout_marginTop="16dp" /> - <com.wuhei.reccoon.LevelMeterView + <com.tomcooks.reccoon.LevelMeterView android:id="@+id/levelMeter" android:layout_width="match_parent" android:layout_height="38dp" android:layout_marginTop="10dp" /> - <Button - android:id="@+id/markButton" + <LinearLayout android:layout_width="wrap_content" android:layout_height="wrap_content" android:layout_gravity="center_horizontal" android:layout_marginTop="6dp" - android:enabled="false" - android:minWidth="140dp" - android:text="@string/mark_action" /> + android:gravity="center" + android:orientation="horizontal"> + + <Button + android:id="@+id/markButton" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:enabled="false" + android:minWidth="130dp" + android:text="@string/mark_action" /> + + <Button + android:id="@+id/discardButton" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:layout_marginStart="10dp" + android:enabled="false" + android:minWidth="130dp" + android:text="@string/discard_action" /> + </LinearLayout> <LinearLayout android:id="@+id/optionsRow" @@ -116,6 +202,17 @@ android:textSize="13sp" /> </LinearLayout> + <androidx.appcompat.widget.AppCompatCheckBox + android:id="@+id/swapCheckbox" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:layout_gravity="center_horizontal" + android:layout_marginTop="2dp" + android:contentDescription="@string/swap_channels" + android:text="@string/swap_short" + android:textColor="@color/text_secondary" + android:textSize="13sp" /> + <TextView android:id="@+id/statusText" android:layout_width="match_parent" diff --git a/app/src/main/res/layout/activity_player.xml b/app/src/main/res/layout/activity_player.xml index 6ba1190..2bb2f54 100644 --- a/app/src/main/res/layout/activity_player.xml +++ b/app/src/main/res/layout/activity_player.xml @@ -159,25 +159,11 @@ android:textSize="18sp" android:textStyle="bold" /> - <LinearLayout + <Spinner + android:id="@+id/modelSpinner" android:layout_width="match_parent" android:layout_height="wrap_content" - android:layout_marginTop="8dp" - android:orientation="horizontal"> - - <Spinner - android:id="@+id/languageSpinner" - android:layout_width="0dp" - android:layout_height="wrap_content" - android:layout_weight="1" /> - - <Spinner - android:id="@+id/modelSpinner" - android:layout_width="0dp" - android:layout_height="wrap_content" - android:layout_marginStart="10dp" - android:layout_weight="1" /> - </LinearLayout> + android:layout_marginTop="8dp" /> <Button android:id="@+id/transcribeButton" diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 416ce5e..6bab640 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1,13 +1,16 @@ <?xml version="1.0" encoding="utf-8"?> <resources> <string name="app_name">RECCoon</string> - <string name="subtitle">by wuhei · v1.4.0-alpha</string> + <string name="subtitle">by Tom Cooks · v1.0.0</string> <string name="timer_zero">00:00</string> <string name="footer">44.1 kHz · 16-bit · PCM (WAV) · cue markers</string> <string name="status_idle">Tap the button to start recording</string> <string name="status_recording">Recording… tap again to stop and save</string> <string name="status_saving">Saving to Downloads…</string> + <string name="status_discarding">Discarding…</string> + <string name="discarded">Recording discarded</string> + <string name="discard_action">Discard</string> <string name="cd_record">Start recording</string> <string name="cd_stop">Stop and save</string> @@ -22,8 +25,22 @@ <string name="mono_short">Mono</string> <string name="mono_recording">Record in mono (half file size)</string> + <string name="compress_short">Compressed (small M4A)</string> + <string name="compress_recording">Save a compressed M4A instead of a WAV (much smaller, lossy)</string> + <string name="input_label">Mic</string> + <string name="input_auto">Auto (Android picks)</string> + <string name="input_phone">Phone mic (built-in)</string> <string name="silence_short">Silence alerts</string> <string name="silence_recording">Silence notifications & vibrations while recording</string> + <string name="swap_short">Swap L/R</string> + <string name="swap_channels">Swap the left and right channels (use this if the meters are inverted)</string> + <string name="monitor_short">Monitor</string> + <string name="monitor_recording">Play the microphone through the selected output while recording</string> + <string name="monitor_warning">Monitoring live audio can cause feedback — use headphones.</string> + <string name="err_monitor">Could not start live monitoring</string> + <string name="output_label">Out</string> + <string name="output_auto">Auto (system default)</string> + <string name="output_speaker">Phone speaker</string> <string name="dnd_title">Allow Do Not Disturb</string> <string name="dnd_message">To silence notifications and vibrations while recording, allow RECCoon access to Do Not Disturb.</string> <string name="dnd_open_settings">Open settings</string> @@ -49,9 +66,10 @@ <string name="export_failed">Export failed: %1$s</string> <string name="err_share">Could not share this file</string> <string name="notification_channel">Recording</string> + <string name="model_download_channel">Model download</string> <string name="notification_recording">Recording audio…</string> <string name="recordings_title">Recordings</string> - <string name="recordings_subtitle">Saved in Downloads · reccoon-*.wav</string> + <string name="recordings_subtitle">Saved in Downloads · reccoon-*.wav / *.m4a</string> <string name="recordings_empty">No recordings yet.\nTap the record button to make one.</string> <string name="badge_transcript">TXT</string> @@ -63,15 +81,11 @@ <string name="transcribe_title">Transcription (on-device)</string> <string name="transcribe_action">Transcribe</string> - <string name="transcribe_hint">Choose a language and model, then tap Transcribe.</string> - <string name="lang_auto">Auto detect</string> - <string name="lang_en">English</string> - <string name="lang_it">Italiano</string> - <string name="model_tiny">Tiny (fast)</string> - <string name="model_base">Base (better)</string> + <string name="transcribe_hint">Tap Transcribe to run Parakeet v3 on this recording.</string> + <string name="model_download_title">Download model</string> - <string name="model_download_message">The %1$s multilingual Whisper model is required (about %2$s). Download it now over Wi-Fi?</string> + <string name="model_download_message">The %1$s model is required (about %2$s). Download it now over Wi-Fi?</string> <string name="download">Download</string> <string name="cancel">Cancel</string> <string name="status_model_ready">Model ready</string> diff --git a/app/src/test/java/com/tomcooks/reccoon/FormatsTest.java b/app/src/test/java/com/tomcooks/reccoon/FormatsTest.java new file mode 100644 index 0000000..96427a1 --- /dev/null +++ b/app/src/test/java/com/tomcooks/reccoon/FormatsTest.java @@ -0,0 +1,20 @@ +package com.tomcooks.reccoon; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class FormatsTest { + + @Test + public void prettyNameStripsWavExtension() { + assertEquals("2026-10-05 09:30:00 #00012", + Formats.prettyName("reccoon-2026-10-05-093000-00012.wav")); + } + + @Test + public void prettyNameStripsM4aExtension() { + assertEquals("2026-10-05 09:30:00 #00012", + Formats.prettyName("reccoon-2026-10-05-093000-00012.m4a")); + } +} diff --git a/app/src/test/java/com/wuhei/reccoon/StreamingResamplerTest.java b/app/src/test/java/com/tomcooks/reccoon/StreamingResamplerTest.java index 6e4cdcd..4c44a44 100644 --- a/app/src/test/java/com/wuhei/reccoon/StreamingResamplerTest.java +++ b/app/src/test/java/com/tomcooks/reccoon/StreamingResamplerTest.java @@ -1,4 +1,4 @@ -package com.wuhei.reccoon; +package com.tomcooks.reccoon; import static org.junit.Assert.assertTrue; diff --git a/settings.gradle.kts b/settings.gradle.kts index 8720732..ca0106d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -23,6 +23,8 @@ dependencyResolutionManagement { } } mavenCentral() + // sherpa-onnx AAR (Apache-2.0). JitPack is trusted by F-Droid. + maven { url = uri("https://jitpack.io") } } } diff --git a/tools/download-parakeet.sh b/tools/download-parakeet.sh new file mode 100755 index 0000000..cb6fafc --- /dev/null +++ b/tools/download-parakeet.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Downloads NVIDIA Parakeet TDT 0.6B v3 (int8) plus the Silero VAD model into a +# folder whose layout matches what RECCoon expects on the device. +# +# Usage: +# ./download-parakeet.sh [output-dir] (default: ./reccoon-models) +# +# Then copy the output directory to the device, e.g.: +# adb push reccoon-models/ /sdcard/Android/data/com.wuhei.reccoon/files/models/ +set -euo pipefail + +OUT="${1:-reccoon-models}" +PARAKEET="https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8/resolve/main" +VAD="https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx" + +mkdir -p "$OUT/parakeet-v3" + +echo "==> Silero VAD" +curl -L --fail -o "$OUT/silero_vad.onnx" "$VAD" + +echo "==> Parakeet v3 (encoder is ~652 MB, this can take a while)" +for file in encoder.int8.onnx decoder.int8.onnx joiner.int8.onnx tokens.txt; do + echo " -> $file" + curl -L --fail -o "$OUT/parakeet-v3/$file" "$PARAKEET/$file" +done + +echo +echo "Done. Layout:" +find "$OUT" -type f -printf ' %p (%s bytes)\n' +echo +echo "On the phone these must live under:" +echo " /sdcard/Android/data/com.wuhei.reccoon/files/models/" +echo "i.e. .../models/silero_vad.onnx and .../models/parakeet-v3/*.onnx" diff --git a/tools/whistle-eval/.gitignore b/tools/whistle-eval/.gitignore new file mode 100644 index 0000000..f28fcce --- /dev/null +++ b/tools/whistle-eval/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +vendor/ +models/ +.venv/ +__pycache__/ +samples/*.wav +samples/*.mp3 diff --git a/tools/whistle-eval/README.md b/tools/whistle-eval/README.md new file mode 100644 index 0000000..ed6e71e --- /dev/null +++ b/tools/whistle-eval/README.md @@ -0,0 +1,94 @@ +# Speech-model evaluation + +Compares on-device speech recognition candidates on the same 16 kHz mono clips: + +- the multilingual Whisper tiny/base models that RECCoon already uses, + through the exact sherpa-onnx 1.13.8 version the Android app bundles; +- [NVIDIA Parakeet TDT 0.6B v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3) + (25 European languages) via its sherpa-onnx int8 export; +- [Cactus Whistle](https://huggingface.co/mrfakename/whistle-ONNX), a custom + *Needle* encoder/decoder run through its reference Node pipeline. + +Whistle is **not** a Whisper model and cannot be loaded with sherpa-onnx. +Parakeet, unlike Whistle, **is** supported by sherpa-onnx. + +## Setup + +Requires Node 18+, Python 3, `ffmpeg`, and `curl`. + +```sh +cd tools/whistle-eval + +# 1. Whistle pack + ONNX graphs + reference JS + onnxruntime-node +# and the Whisper tiny/base int8 models. +./setup.sh + +# 2. Parakeet v3 int8 (~660 MB, optional) +mkdir -p models/parakeet-v3-int8 +BASE=https://huggingface.co/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8/resolve/main +for f in encoder.int8.onnx decoder.int8.onnx joiner.int8.onnx tokens.txt; do + curl -L --fail -o "models/parakeet-v3-int8/$f" "$BASE/$f" +done + +# 3. Python environment for the sherpa-onnx baselines. +python3 -m venv .venv +.venv/bin/pip install sherpa-onnx numpy +``` + +`setup.sh` downloads the 17 MB `whistle.pack` and rebuilds the fp32 +`encoder.onnx.data` / `decoder.onnx.data` files locally with `build-data.mjs`. +They are **not** committed (see `.gitignore`); the pack is enough to recreate +them byte-exactly. + +## Run + +```sh +# One clip (language inferred: `it*` -> Italian, otherwise English) +.venv/bin/python evaluate.py samples/jfk.wav + +# A whole folder +.venv/bin/python evaluate.py "samples/*.wav" +``` + +`evaluate.py` prints the transcript and the word error rate (WER) against +`samples/<name>.txt` when a reference exists. Parakeet is included +automatically when `models/parakeet-v3-int8/` is present. + +| clip | Whisper tiny | Whisper base | Whistle | **Parakeet v3** | +|---|---|---|---|---| +| `jfk.wav` | 0% | 4.5% | 0% | 0% | +| `en1.wav` | 0% | 0% | 8.7% | 0% | +| `en2.wav` | 13.6% | 13.6% | 13.6% | 13.6% | +| `it1.wav` | 24% | 12% | 20% | **4%** | +| `it1_noisy.wav` | 24% | 12% | 24% | **0%** | +| `it2.wav` | 21.7% | 26.1% | 4.3% | 4.3% | + +Average WER over these six clips (lower is better): + +| model | English (3) | Italian (3) | All (6) | int8 size | +|---|---|---|---|---| +| Whisper tiny | 4.5% | 23.2% | 13.9% | ~103 MB | +| Whisper base | 6.0% | 16.7% | 11.4% | ~160 MB | +| Whistle | 7.4% | 16.1% | 11.8% | ~17 MB pack (desktop) | +| **Parakeet v3** | 4.5% | **2.8%** | **3.7%** | ~670 MB | + +### Reading the result + +- **Parakeet v3 is the clear winner**, especially for Italian and noisy audio: + it was perfect on the noisy Italian clip that fooled every other model. +- The only Parakeet "errors" are number formatting ("four fifteen" → "4:15"), + which WER counts even though the transcript is correct. +- The cost is size: ~670 MB int8, about four times Whisper base. It is usable + on a modern phone but is a large download and a lot of RAM. +- Whisper base stays the best small model; Whistle is competitive but not a + clear win, and needs a custom runtime. +- These are small neural-TTS clips, not real recordings. Run the harness on + real RECCoon recordings before tuning further. + +## Android notes + +- Parakeet can be added through the existing sherpa-onnx AAR with + `model_type="nemo_transducer"` (Java: `OfflineTransducerModelConfig`). RECCoon + 1.5.0-alpha exposes it as the "Parakeet v3 (best)" model in the player. +- Whistle would need `onnxruntime-android` plus the log-mel frontend, BPE + tokenizer, engram features and greedy decoder from `vendor/js/whistle.js`. diff --git a/tools/whistle-eval/build-data.mjs b/tools/whistle-eval/build-data.mjs new file mode 100644 index 0000000..20be062 --- /dev/null +++ b/tools/whistle-eval/build-data.mjs @@ -0,0 +1,19 @@ +// Rebuilds the fp32 ONNX external-data files from the compact whistle.pack. +// This is the "small weights" path: instead of committing ~145 MB of .onnx.data +// to git (or downloading it), keep the 17 MB pack and expand it on first run. +import fs from 'fs'; +import { parsePack, buildModelData } from './vendor/js/pack.js'; + +const root = new URL('./vendor/', import.meta.url); +const packBytes = fs.readFileSync(new URL('whistle.pack', root)); +const pack = parsePack( + packBytes.buffer.slice(packBytes.byteOffset, packBytes.byteOffset + packBytes.byteLength)); + +for (const name of ['encoder', 'decoder']) { + const target = new URL(`onnx/${name}.onnx.data`, root); + const data = buildModelData(pack, name, (p) => { + process.stdout.write(`\r${name}: ${(p * 100).toFixed(0)}% `); + }); + fs.writeFileSync(target, data); + process.stdout.write(`\r${name}: wrote ${data.length} bytes\n`); +} diff --git a/tools/whistle-eval/evaluate.py b/tools/whistle-eval/evaluate.py new file mode 100644 index 0000000..f4437ea --- /dev/null +++ b/tools/whistle-eval/evaluate.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Compare Cactus Whistle with the app's Whisper models on the same clips. + +Usage: + python evaluate.py samples/jfk.wav en + python evaluate.py samples/*.wav # language guessed from a *.it.wav suffix + +For every clip this runs: + * Whisper tiny (int8, multilingual) through sherpa-onnx + * Whisper base (int8, multilingual) through sherpa-onnx + * Cactus Whistle through its reference Node pipeline +and prints the transcript plus the word error rate (WER) against +`samples/<name>.txt` when that reference file exists. + +Setup (see README.md): + npm install + node build-data.mjs + python3 -m venv .venv && .venv/bin/pip install sherpa-onnx numpy +""" +import argparse +import glob +import os +import re +import subprocess +import sys +import unicodedata +import wave + +HERE = os.path.dirname(os.path.abspath(__file__)) +VENV = os.environ.get("WHISTLE_EVAL_PYTHON", sys.executable) + + +def normalize(text: str) -> list: + text = unicodedata.normalize("NFKD", text.lower()) + text = "".join(c for c in text if not unicodedata.combining(c)) + text = re.sub(r"[^a-z0-9]+", " ", text) + return text.split() + + +def wer(reference: list, hypothesis: list) -> float: + if not reference: + return 0.0 if not hypothesis else 1.0 + previous = list(range(len(hypothesis) + 1)) + for i, ref in enumerate(reference, start=1): + current = [i] + for j, hyp in enumerate(hypothesis, start=1): + cost = 0 if ref == hyp else 1 + current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + cost)) + previous = current + return previous[-1] / len(reference) + + +def read_reference(audio: str): + base = os.path.splitext(audio)[0] + for candidate in (base + ".txt", audio + ".txt"): + if os.path.isfile(candidate): + with open(candidate, encoding="utf-8") as handle: + return handle.read().strip() + return None + + +def run_whistle(audio: str) -> str: + out = subprocess.run( + ["node", os.path.join(HERE, "vendor", "js", "example.mjs"), audio], + cwd=HERE, capture_output=True, text=True, timeout=600) + if out.returncode != 0: + return f"<whistle failed: {out.stderr.strip().splitlines()[-1] if out.stderr else '?'}>" + first = out.stdout.strip().splitlines()[0] if out.stdout.strip() else "" + return re.sub(r"^\[[a-z]{2}\]\s*", "", first) + + +def parakeet_available() -> bool: + return os.path.isfile(os.path.join( + HERE, "models", "parakeet-v3-int8", "encoder.int8.onnx")) + + +def run_parakeet(audio: str) -> str: + script = os.path.join(HERE, "parakeet_baseline.py") + out = subprocess.run( + [VENV, script, audio], capture_output=True, text=True, timeout=1200) + if out.returncode != 0: + tail = out.stderr.strip().splitlines() + return f"<parakeet failed: {tail[-1] if tail else '?'}>" + return out.stdout.strip() + + +def run_whisper(audio: str, size: str, language: str) -> str: + script = os.path.join(HERE, "whisper_baseline.py") + out = subprocess.run( + [VENV, script, audio, size, language], + capture_output=True, text=True, timeout=900) + if out.returncode != 0: + tail = out.stderr.strip().splitlines() + return f"<whisper {size} failed: {tail[-1] if tail else '?'}>" + return out.stdout.strip() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("clips", nargs="+") + parser.add_argument("language", nargs="?", default=None) + args = parser.parse_args() + + clips = [] + for pattern in args.clips: + clips.extend(sorted(glob.glob(pattern)) or [pattern]) + + print(f"{'clip':<16} {'model':<14} {'WER':>7} transcript") + print("-" * 100) + for audio in clips: + if not os.path.isfile(audio): + print(f"{audio}: not found") + continue + language = args.language + name = os.path.basename(audio).lower() + if language is None: + language = "it" if (name.startswith("it") or ".it." in name) else "en" + reference = read_reference(audio) + ref_words = normalize(reference) if reference else None + + results = [ + ("whistle", run_whistle(audio)), + ("whisper tiny", run_whisper(audio, "tiny", language)), + ("whisper base", run_whisper(audio, "base", language)), + ] + if parakeet_available(): + results.append(("parakeet v3", run_parakeet(audio))) + for model, text in results: + if ref_words is None: + score = " n/a" + else: + score = f"{wer(ref_words, normalize(text)) * 100:6.1f}%" + print(f"{os.path.basename(audio):<16} {model:<14} {score} {text}") + if reference: + print(f"{'':<16} {'reference':<14} {'':>7} {reference}") + print() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/whistle-eval/package-lock.json b/tools/whistle-eval/package-lock.json new file mode 100644 index 0000000..063b6cc --- /dev/null +++ b/tools/whistle-eval/package-lock.json @@ -0,0 +1,229 @@ +{ + "name": "reccoon-whistle-eval", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "reccoon-whistle-eval", + "version": "1.0.0", + "dependencies": { + "onnxruntime-node": "^1.20.1" + } + }, + "node_modules/adm-zip": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", + "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", + "license": "MIT", + "engines": { + "node": ">=14.0" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/global-agent": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-4.1.3.tgz", + "integrity": "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==", + "license": "BSD-3-Clause", + "dependencies": { + "globalthis": "^1.0.2", + "matcher": "^4.0.0", + "semver": "^7.3.5", + "serialize-error": "^8.1.0" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/matcher": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-4.0.0.tgz", + "integrity": "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz", + "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.30.0.tgz", + "integrity": "sha512-twhs1C2C/BFkz1yc5OY0KIU2GUq6DURO7hD4bx5Q2Qy3nAMJwRXW8xU3NVczE29VA9lolLOYepoD8fjTGOfIqw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.6.0", + "global-agent": "^4.1.3", + "onnxruntime-common": "1.30.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/tools/whistle-eval/package.json b/tools/whistle-eval/package.json new file mode 100644 index 0000000..b34444b --- /dev/null +++ b/tools/whistle-eval/package.json @@ -0,0 +1,14 @@ +{ + "name": "reccoon-whistle-eval", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "Desktop evaluation harness for the Cactus Whistle ASR model.", + "scripts": { + "build-data": "node build-data.mjs", + "transcribe": "node vendor/js/example.mjs" + }, + "dependencies": { + "onnxruntime-node": "^1.20.1" + } +} diff --git a/tools/whistle-eval/parakeet_baseline.py b/tools/whistle-eval/parakeet_baseline.py new file mode 100644 index 0000000..16987de --- /dev/null +++ b/tools/whistle-eval/parakeet_baseline.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Run NVIDIA Parakeet TDT 0.6B v3 (multilingual) through sherpa-onnx. + +Usage: + python parakeet_baseline.py <audio.wav> + +Requires a virtualenv with `sherpa-onnx` installed and the int8 model under +./models/parakeet-v3-int8/ (see README.md). +""" +import argparse +import os +import sys +import wave + +import numpy as np +import sherpa_onnx + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("audio") + args = parser.parse_args() + + model_dir = os.path.join(os.path.dirname(__file__), "models", "parakeet-v3-int8") + encoder = os.path.join(model_dir, "encoder.int8.onnx") + decoder = os.path.join(model_dir, "decoder.int8.onnx") + joiner = os.path.join(model_dir, "joiner.int8.onnx") + tokens = os.path.join(model_dir, "tokens.txt") + for path in (encoder, decoder, joiner, tokens): + if not os.path.isfile(path): + print(f"missing {path}", file=sys.stderr) + return 2 + + recognizer = sherpa_onnx.OfflineRecognizer.from_transducer( + encoder=encoder, + decoder=decoder, + joiner=joiner, + tokens=tokens, + num_threads=2, + sample_rate=16000, + feature_dim=80, + decoding_method="greedy_search", + model_type="nemo_transducer", + debug=False, + ) + + with wave.open(args.audio, "rb") as wav: + sample_rate = wav.getframerate() + channels = wav.getnchannels() + width = wav.getsampwidth() + frames = wav.readframes(wav.getnframes()) + if width != 2: + print("only 16-bit PCM WAV is supported", file=sys.stderr) + return 2 + samples = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0 + if channels > 1: + samples = samples.reshape(-1, channels).mean(axis=1) + + stream = recognizer.create_stream() + stream.accept_waveform(sample_rate, samples) + recognizer.decode_stream(stream) + print(stream.result.text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/whistle-eval/samples/README.md b/tools/whistle-eval/samples/README.md new file mode 100644 index 0000000..28f0d12 --- /dev/null +++ b/tools/whistle-eval/samples/README.md @@ -0,0 +1,41 @@ +# Test clips + +The audio files are not committed. Recreate them with `edge-tts` (neural TTS) +and `ffmpeg`, or drop your own 16 kHz mono 16-bit WAV files here. When a +`<name>.txt` sidecar exists, `evaluate.py` reports the WER against it. + +```sh +# English +edge-tts --voice en-US-AriaNeural \ + --text "The quick brown fox jumps over the lazy dog near the riverbank, while the scientists carefully measured the temperature of the experimental apparatus." \ + --write-media en1.mp3 +# Italian +edge-tts --voice it-IT-ElsaNeural \ + --text "Il rapido volpe marrone salta sopra il cane pigro vicino alla riva del fiume, mentre gli scienziati misurano con attenzione la temperatura dell'apparecchiatura sperimentale." \ + --write-media it1.mp3 +# ...then: +for f in en1 it1; do + ffmpeg -y -i "$f.mp3" -ar 16000 -ac 1 -c:a pcm_s16le "$f.wav" +done +``` + +The JFK clip is the public sample from +[whisper.cpp](https://github.com/ggerganov/whisper.cpp/blob/master/samples/jfk.wav). + +## References used by the committed test set + +| file | reference | +|---|---| +| `jfk.wav` | And so my fellow Americans ask not what your country can do for you ask what you can do for your country | +| `en1.wav` | The quick brown fox jumps over the lazy dog near the riverbank, while the scientists carefully measured the temperature of the experimental apparatus. | +| `en2.wav` | Please call me back at four fifteen in the afternoon. The meeting is with Doctor Smith and Professor Johnson from the university. | +| `it1.wav` | Il rapido volpe marrone salta sopra il cane pigro vicino alla riva del fiume, mentre gli scienziati misurano con attenzione la temperatura dell'apparecchiatura sperimentale. | +| `it2.wav` | Per favore, richiamami alle quattro e un quarto del pomeriggio. L'incontro e con il dottor Rossi e la professoressa Bianchi dell'universita. | + +`it1_noisy.wav` is `it1.wav` mixed with low-level white noise: + +```sh +ffmpeg -y -i it1.wav -f lavfi -i "anoisesrc=color=white:amplitude=0.03:duration=999" \ + -filter_complex "[0:a][1:a]amix=inputs=2:duration=first:weights=1 0.35" \ + -ar 16000 -ac 1 -c:a pcm_s16le it1_noisy.wav +``` diff --git a/tools/whistle-eval/samples/en1.txt b/tools/whistle-eval/samples/en1.txt new file mode 100644 index 0000000..2848fe3 --- /dev/null +++ b/tools/whistle-eval/samples/en1.txt @@ -0,0 +1 @@ +The quick brown fox jumps over the lazy dog near the riverbank, while the scientists carefully measured the temperature of the experimental apparatus. diff --git a/tools/whistle-eval/samples/en2.txt b/tools/whistle-eval/samples/en2.txt new file mode 100644 index 0000000..da8a6ff --- /dev/null +++ b/tools/whistle-eval/samples/en2.txt @@ -0,0 +1 @@ +Please call me back at four fifteen in the afternoon. The meeting is with Doctor Smith and Professor Johnson from the university. diff --git a/tools/whistle-eval/samples/it1.txt b/tools/whistle-eval/samples/it1.txt new file mode 100644 index 0000000..680e8da --- /dev/null +++ b/tools/whistle-eval/samples/it1.txt @@ -0,0 +1 @@ +Il rapido volpe marrone salta sopra il cane pigro vicino alla riva del fiume, mentre gli scienziati misurano con attenzione la temperatura dell'apparecchiatura sperimentale. diff --git a/tools/whistle-eval/samples/it1_noisy.txt b/tools/whistle-eval/samples/it1_noisy.txt new file mode 100644 index 0000000..680e8da --- /dev/null +++ b/tools/whistle-eval/samples/it1_noisy.txt @@ -0,0 +1 @@ +Il rapido volpe marrone salta sopra il cane pigro vicino alla riva del fiume, mentre gli scienziati misurano con attenzione la temperatura dell'apparecchiatura sperimentale. diff --git a/tools/whistle-eval/samples/it2.txt b/tools/whistle-eval/samples/it2.txt new file mode 100644 index 0000000..903d626 --- /dev/null +++ b/tools/whistle-eval/samples/it2.txt @@ -0,0 +1 @@ +Per favore, richiamami alle quattro e un quarto del pomeriggio. L'incontro e con il dottor Rossi e la professoressa Bianchi dell'universita. diff --git a/tools/whistle-eval/samples/jfk.txt b/tools/whistle-eval/samples/jfk.txt new file mode 100644 index 0000000..a67a12c --- /dev/null +++ b/tools/whistle-eval/samples/jfk.txt @@ -0,0 +1 @@ +And so my fellow Americans ask not what your country can do for you ask what you can do for your country diff --git a/tools/whistle-eval/setup.sh b/tools/whistle-eval/setup.sh new file mode 100755 index 0000000..fde74fe --- /dev/null +++ b/tools/whistle-eval/setup.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Fetches everything the Whistle evaluation harness needs and rebuilds the +# ONNX external-data files from the small whistle.pack. +set -euo pipefail +cd "$(dirname "$0")" + +WHISTLE_BASE="https://huggingface.co/mrfakename/whistle-ONNX/resolve/main" +WHISPER_BASE="https://huggingface.co/csukuangfj/sherpa-onnx-whisper" + +mkdir -p vendor/js vendor/onnx models/tiny models/base + +echo "==> Downloading the Whistle pack and ONNX graphs" +curl -L --fail -o vendor/whistle.pack "$WHISTLE_BASE/whistle.pack" +curl -L --fail -o vendor/onnx/encoder.onnx "$WHISTLE_BASE/onnx/encoder.onnx" +curl -L --fail -o vendor/onnx/decoder.onnx "$WHISTLE_BASE/onnx/decoder.onnx" + +echo "==> Downloading the reference JS pipeline" +for file in pack.js whistle.js backend.js example.mjs; do + curl -L --fail -o "vendor/js/$file" "$WHISTLE_BASE/js/$file" +done + +echo "==> Installing onnxruntime-node" +npm install --no-audit --no-fund + +echo "==> Rebuilding fp32 ONNX data from whistle.pack" +node build-data.mjs + +echo "==> Downloading Whisper tiny/base int8 (sherpa-onnx)" +for model in tiny base; do + for file in encoder.int8.onnx decoder.int8.onnx tokens.txt; do + curl -L --fail -o "models/$model/$file" "$WHISPER_BASE-$model/resolve/main/$model-$file" + done +done + +echo +echo "Done. Next:" +echo " python3 -m venv .venv && .venv/bin/pip install sherpa-onnx numpy" +echo " .venv/bin/python evaluate.py \"samples/*.wav\"" diff --git a/tools/whistle-eval/whisper_baseline.py b/tools/whistle-eval/whisper_baseline.py new file mode 100644 index 0000000..9be7014 --- /dev/null +++ b/tools/whistle-eval/whisper_baseline.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Run the same sherpa-onnx multilingual Whisper model that the app uses. + +Usage: + python whisper_baseline.py <audio.wav> [tiny|base] [language] + +Requires a virtualenv with `sherpa-onnx` installed (see README.md). The model +files are expected under ./models/<size>/. +""" +import argparse +import os +import sys +import wave + +import numpy as np +import sherpa_onnx + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("audio") + parser.add_argument("size", nargs="?", default="base", choices=["tiny", "base"]) + parser.add_argument("language", nargs="?", default="en") + args = parser.parse_args() + + model_dir = os.path.join(os.path.dirname(__file__), "models", args.size) + encoder = os.path.join(model_dir, "encoder.int8.onnx") + decoder = os.path.join(model_dir, "decoder.int8.onnx") + tokens = os.path.join(model_dir, "tokens.txt") + for path in (encoder, decoder, tokens): + if not os.path.isfile(path): + print(f"missing {path}", file=sys.stderr) + return 2 + + recognizer = sherpa_onnx.OfflineRecognizer.from_whisper( + encoder=encoder, + decoder=decoder, + tokens=tokens, + language=args.language, + task="transcribe", + num_threads=2, + decoding_method="greedy_search", + debug=False, + ) + with wave.open(args.audio, "rb") as wav: + sample_rate = wav.getframerate() + channels = wav.getnchannels() + width = wav.getsampwidth() + frames = wav.readframes(wav.getnframes()) + if width != 2: + print("only 16-bit PCM WAV is supported", file=sys.stderr) + return 2 + samples = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0 + if channels > 1: + samples = samples.reshape(-1, channels).mean(axis=1) + + stream = recognizer.create_stream() + stream.accept_waveform(sample_rate, samples) + recognizer.decode_stream(stream) + print(stream.result.text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) |
